From baf282ecb22789c27f886e581632692d22472ac2 Mon Sep 17 00:00:00 2001 From: xushiwei Date: Sat, 15 Jun 2024 20:46:29 +0800 Subject: [PATCH 1/6] packages.LoadEx: support Deduper --- internal/build/build.go | 25 ++----- internal/build/clean.go | 2 +- internal/llgen/llgenf.go | 4 +- internal/packages/load.go | 139 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 142 insertions(+), 28 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 1a130122..d2a77cbb 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -28,7 +28,6 @@ import ( "path/filepath" "runtime" "strings" - "unsafe" "golang.org/x/tools/go/ssa" @@ -107,11 +106,13 @@ func Do(args []string, conf *Config) { prog := llssa.NewProgram(nil) sizes := prog.TypeSizes + // dedup := packages.NewDeduper() + dedup := (*packages.Deduper)(nil) if patterns == nil { patterns = []string{"."} } - initial, err := packages.LoadEx(sizes, cfg, patterns...) + initial, err := packages.LoadEx(dedup, sizes, cfg, patterns...) check(err) mode := conf.Mode @@ -133,7 +134,7 @@ func Do(args []string, conf *Config) { load := func() []*packages.Package { if rt == nil { var err error - rt, err = packages.LoadEx(sizes, cfg, llssa.PkgRuntime, llssa.PkgPython) + rt, err = packages.LoadEx(dedup, sizes, cfg, llssa.PkgRuntime, llssa.PkgPython) check(err) } return rt @@ -149,7 +150,7 @@ func Do(args []string, conf *Config) { }) imp := func(pkgPath string) *packages.Package { - if ret, e := packages.LoadEx(sizes, cfg, pkgPath); e == nil { + if ret, e := packages.LoadEx(dedup, sizes, cfg, pkgPath); e == nil { return ret[0] } return nil @@ -443,17 +444,6 @@ func allPkgs(imp importer, initial []*packages.Package, mode ssa.BuilderMode) (p return } -type ssaProgram struct { - Fset *token.FileSet - imported map[string]*ssa.Package - packages map[*types.Package]*ssa.Package // TODO(xsw): ensure offset of packages -} - -func setPkgSSA(prog *ssa.Program, pkg *types.Package, pkgSSA *ssa.Package) { - s := (*ssaProgram)(unsafe.Pointer(prog)) - s.packages[pkg] = pkgSSA -} - func createAltSSAPkg(prog *ssa.Program, alt *packages.Package) *ssa.Package { altPath := alt.Types.Path() altSSA := prog.ImportedPackage(altPath) @@ -461,11 +451,8 @@ func createAltSSAPkg(prog *ssa.Program, alt *packages.Package) *ssa.Package { packages.Visit([]*packages.Package{alt}, nil, func(p *packages.Package) { pkgTypes := p.Types if pkgTypes != nil && !p.IllTyped { - pkgSSA := prog.ImportedPackage(pkgTypes.Path()) - if pkgSSA == nil { + if prog.ImportedPackage(pkgTypes.Path()) == nil { prog.CreatePackage(pkgTypes, p.Syntax, p.TypesInfo, true) - } else { - setPkgSSA(prog, pkgTypes, pkgSSA) } } }) diff --git a/internal/build/clean.go b/internal/build/clean.go index 1e5a2e26..60ec5f46 100644 --- a/internal/build/clean.go +++ b/internal/build/clean.go @@ -42,7 +42,7 @@ func Clean(args []string, conf *Config) { if patterns == nil { patterns = []string{"."} } - initial, err := packages.LoadEx(nil, cfg, patterns...) + initial, err := packages.LoadEx(nil, nil, cfg, patterns...) check(err) cleanPkgs(initial, verbose) diff --git a/internal/llgen/llgenf.go b/internal/llgen/llgenf.go index 843c245a..916831dc 100644 --- a/internal/llgen/llgenf.go +++ b/internal/llgen/llgenf.go @@ -43,7 +43,7 @@ func initRtAndPy(prog llssa.Program, cfg *packages.Config) { load := func() []*packages.Package { if pkgRtAndPy == nil { var err error - pkgRtAndPy, err = packages.LoadEx(prog.TypeSizes, cfg, llssa.PkgRuntime, llssa.PkgPython) + pkgRtAndPy, err = packages.LoadEx(nil, prog.TypeSizes, cfg, llssa.PkgRuntime, llssa.PkgPython) check(err) } return pkgRtAndPy @@ -65,7 +65,7 @@ func GenFrom(fileOrPkg string) string { cfg := &packages.Config{ Mode: loadSyntax | packages.NeedDeps, } - initial, err := packages.LoadEx(prog.TypeSizes, cfg, fileOrPkg) + initial, err := packages.LoadEx(nil, prog.TypeSizes, cfg, fileOrPkg) check(err) _, pkgs := ssautil.AllPackages(initial, ssa.SanityCheckFunctions) diff --git a/internal/packages/load.go b/internal/packages/load.go index 18d49d63..22e4fbb6 100644 --- a/internal/packages/load.go +++ b/internal/packages/load.go @@ -17,12 +17,14 @@ package packages import ( + "errors" "fmt" "go/types" "runtime" "sync" "unsafe" + "golang.org/x/sync/errgroup" "golang.org/x/tools/go/packages" ) @@ -57,6 +59,12 @@ const ( // Calls to Load do not modify this struct. type Config = packages.Config +func setGoListOverlayFile(cfg *Config, val string) { + // TODO(xsw): suppose that the field is at the end of the struct + ptr := uintptr(unsafe.Pointer(cfg)) + (unsafe.Sizeof(*cfg) - unsafe.Sizeof(val)) + *(*string)(unsafe.Pointer(ptr)) = val +} + // A Package describes a loaded Go package. type Package = packages.Package @@ -64,7 +72,7 @@ type Package = packages.Package type loader struct { pkgs map[string]unsafe.Pointer Config - sizes types.Sizes // non-nil if needed by mode + sizes types.Sizes // TODO(xsw): ensure offset of sizes parseCache map[string]unsafe.Pointer parseCacheMu sync.Mutex exportMu sync.Mutex // enforces mutual exclusion of exportdata operations @@ -78,12 +86,131 @@ type loader struct { requestedMode LoadMode } +// Deduper wraps a DriverResponse, deduplicating its contents. +type Deduper struct { + seenRoots map[string]bool + seenPackages map[string]*Package + dr *packages.DriverResponse // TODO(xsw): ensure offset of dr +} + +//go:linkname NewDeduper golang.org/x/tools/go/packages.newDeduper +func NewDeduper() *Deduper + +//go:linkname addAll golang.org/x/tools/go/packages.(*responseDeduper).addAll +func addAll(r *Deduper, dr *packages.DriverResponse) + +func mergeResponsesEx(dedup *Deduper, responses ...*packages.DriverResponse) *packages.DriverResponse { + if len(responses) == 0 { + return nil + } + if dedup == nil { + dedup = NewDeduper() + } + response := dedup + response.dr.NotHandled = false + response.dr.Compiler = responses[0].Compiler + response.dr.Arch = responses[0].Arch + response.dr.GoVersion = responses[0].GoVersion + for _, v := range responses { + addAll(response, v) + } + return response.dr +} + +// driver is the type for functions that query the build system for the +// packages named by the patterns. +type driver func(cfg *Config, patterns ...string) (*packages.DriverResponse, error) + +func callDriverOnChunksEx(dedup *Deduper, driver driver, cfg *Config, chunks [][]string) (*packages.DriverResponse, error) { + if len(chunks) == 0 { + return driver(cfg) + } + responses := make([]*packages.DriverResponse, len(chunks)) + errNotHandled := errors.New("driver returned NotHandled") + var g errgroup.Group + for i, chunk := range chunks { + i := i + chunk := chunk + g.Go(func() (err error) { + responses[i], err = driver(cfg, chunk...) + if responses[i] != nil && responses[i].NotHandled { + err = errNotHandled + } + return err + }) + } + if err := g.Wait(); err != nil { + if errors.Is(err, errNotHandled) { + return &packages.DriverResponse{NotHandled: true}, nil + } + return nil, err + } + return mergeResponsesEx(dedup, responses...), nil +} + +//go:linkname splitIntoChunks golang.org/x/tools/go/packages.splitIntoChunks +func splitIntoChunks(patterns []string, argMax int) ([][]string, error) + +//go:linkname findExternalDriver golang.org/x/tools/go/packages.findExternalDriver +func findExternalDriver(cfg *Config) driver + +//go:linkname goListDriver golang.org/x/tools/go/packages.goListDriver +func goListDriver(cfg *Config, patterns ...string) (_ *packages.DriverResponse, err error) + +//go:linkname writeOverlays golang.org/x/tools/internal/gocommand.WriteOverlays +func writeOverlays(overlay map[string][]byte) (filename string, cleanup func(), err error) + +func defaultDriverEx(dedup *Deduper, cfg *Config, patterns ...string) (*packages.DriverResponse, bool, error) { + const ( + // windowsArgMax specifies the maximum command line length for + // the Windows' CreateProcess function. + windowsArgMax = 32767 + // maxEnvSize is a very rough estimation of the maximum environment + // size of a user. + maxEnvSize = 16384 + // safeArgMax specifies the maximum safe command line length to use + // by the underlying driver excl. the environment. We choose the Windows' + // ARG_MAX as the starting point because it's one of the lowest ARG_MAX + // constants out of the different supported platforms, + // e.g., https://www.in-ulm.de/~mascheck/various/argmax/#results. + safeArgMax = windowsArgMax - maxEnvSize + ) + chunks, err := splitIntoChunks(patterns, safeArgMax) + if err != nil { + return nil, false, err + } + + if driver := findExternalDriver(cfg); driver != nil { + response, err := callDriverOnChunksEx(dedup, driver, cfg, chunks) + if err != nil { + return nil, false, err + } else if !response.NotHandled { + return response, true, nil + } + // (fall through) + } + + // go list fallback + // + // Write overlays once, as there are many calls + // to 'go list' (one per chunk plus others too). + overlay, cleanupOverlay, err := writeOverlays(cfg.Overlay) + if err != nil { + return nil, false, err + } + defer cleanupOverlay() + setGoListOverlayFile(cfg, overlay) + + response, err := callDriverOnChunksEx(dedup, goListDriver, cfg, chunks) + if err != nil { + return nil, false, err + } + return response, false, err +} + //go:linkname newLoader golang.org/x/tools/go/packages.newLoader func newLoader(cfg *Config) *loader -//go:linkname defaultDriver golang.org/x/tools/go/packages.defaultDriver -func defaultDriver(cfg *Config, patterns ...string) (*packages.DriverResponse, bool, error) - //go:linkname refine golang.org/x/tools/go/packages.(*loader).refine func refine(ld *loader, response *packages.DriverResponse) ([]*Package, error) @@ -101,9 +228,9 @@ func refine(ld *loader, response *packages.DriverResponse) ([]*Package, error) // return an error. Clients may need to handle such errors before // proceeding with further analysis. The PrintErrors function is // provided for convenient display of all errors. -func LoadEx(sizes func(types.Sizes) types.Sizes, cfg *Config, patterns ...string) ([]*Package, error) { +func LoadEx(dedup *Deduper, sizes func(types.Sizes) types.Sizes, cfg *Config, patterns ...string) ([]*Package, error) { ld := newLoader(cfg) - response, external, err := defaultDriver(&ld.Config, patterns...) + response, external, err := defaultDriverEx(dedup, &ld.Config, patterns...) if err != nil { return nil, err } From 09e1f9addf69825b96c1be5e855bc57a205cc44a Mon Sep 17 00:00:00 2001 From: xushiwei Date: Sat, 15 Jun 2024 21:12:43 +0800 Subject: [PATCH 2/6] rm defaultDriverEx --- internal/build/build.go | 3 +- internal/packages/load.go | 130 ++------------------------------------ 2 files changed, 6 insertions(+), 127 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index d2a77cbb..f283130a 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -106,8 +106,7 @@ func Do(args []string, conf *Config) { prog := llssa.NewProgram(nil) sizes := prog.TypeSizes - // dedup := packages.NewDeduper() - dedup := (*packages.Deduper)(nil) + dedup := packages.NewDeduper() if patterns == nil { patterns = []string{"."} diff --git a/internal/packages/load.go b/internal/packages/load.go index 22e4fbb6..49b81d8f 100644 --- a/internal/packages/load.go +++ b/internal/packages/load.go @@ -17,14 +17,12 @@ package packages import ( - "errors" "fmt" "go/types" "runtime" "sync" "unsafe" - "golang.org/x/sync/errgroup" "golang.org/x/tools/go/packages" ) @@ -59,12 +57,6 @@ const ( // Calls to Load do not modify this struct. type Config = packages.Config -func setGoListOverlayFile(cfg *Config, val string) { - // TODO(xsw): suppose that the field is at the end of the struct - ptr := uintptr(unsafe.Pointer(cfg)) + (unsafe.Sizeof(*cfg) - unsafe.Sizeof(val)) - *(*string)(unsafe.Pointer(ptr)) = val -} - // A Package describes a loaded Go package. type Package = packages.Package @@ -86,127 +78,15 @@ type loader struct { requestedMode LoadMode } -// Deduper wraps a DriverResponse, deduplicating its contents. type Deduper struct { - seenRoots map[string]bool - seenPackages map[string]*Package - dr *packages.DriverResponse // TODO(xsw): ensure offset of dr } -//go:linkname NewDeduper golang.org/x/tools/go/packages.newDeduper -func NewDeduper() *Deduper - -//go:linkname addAll golang.org/x/tools/go/packages.(*responseDeduper).addAll -func addAll(r *Deduper, dr *packages.DriverResponse) - -func mergeResponsesEx(dedup *Deduper, responses ...*packages.DriverResponse) *packages.DriverResponse { - if len(responses) == 0 { - return nil - } - if dedup == nil { - dedup = NewDeduper() - } - response := dedup - response.dr.NotHandled = false - response.dr.Compiler = responses[0].Compiler - response.dr.Arch = responses[0].Arch - response.dr.GoVersion = responses[0].GoVersion - for _, v := range responses { - addAll(response, v) - } - return response.dr +func NewDeduper() *Deduper { + return nil } -// driver is the type for functions that query the build system for the -// packages named by the patterns. -type driver func(cfg *Config, patterns ...string) (*packages.DriverResponse, error) - -func callDriverOnChunksEx(dedup *Deduper, driver driver, cfg *Config, chunks [][]string) (*packages.DriverResponse, error) { - if len(chunks) == 0 { - return driver(cfg) - } - responses := make([]*packages.DriverResponse, len(chunks)) - errNotHandled := errors.New("driver returned NotHandled") - var g errgroup.Group - for i, chunk := range chunks { - i := i - chunk := chunk - g.Go(func() (err error) { - responses[i], err = driver(cfg, chunk...) - if responses[i] != nil && responses[i].NotHandled { - err = errNotHandled - } - return err - }) - } - if err := g.Wait(); err != nil { - if errors.Is(err, errNotHandled) { - return &packages.DriverResponse{NotHandled: true}, nil - } - return nil, err - } - return mergeResponsesEx(dedup, responses...), nil -} - -//go:linkname splitIntoChunks golang.org/x/tools/go/packages.splitIntoChunks -func splitIntoChunks(patterns []string, argMax int) ([][]string, error) - -//go:linkname findExternalDriver golang.org/x/tools/go/packages.findExternalDriver -func findExternalDriver(cfg *Config) driver - -//go:linkname goListDriver golang.org/x/tools/go/packages.goListDriver -func goListDriver(cfg *Config, patterns ...string) (_ *packages.DriverResponse, err error) - -//go:linkname writeOverlays golang.org/x/tools/internal/gocommand.WriteOverlays -func writeOverlays(overlay map[string][]byte) (filename string, cleanup func(), err error) - -func defaultDriverEx(dedup *Deduper, cfg *Config, patterns ...string) (*packages.DriverResponse, bool, error) { - const ( - // windowsArgMax specifies the maximum command line length for - // the Windows' CreateProcess function. - windowsArgMax = 32767 - // maxEnvSize is a very rough estimation of the maximum environment - // size of a user. - maxEnvSize = 16384 - // safeArgMax specifies the maximum safe command line length to use - // by the underlying driver excl. the environment. We choose the Windows' - // ARG_MAX as the starting point because it's one of the lowest ARG_MAX - // constants out of the different supported platforms, - // e.g., https://www.in-ulm.de/~mascheck/various/argmax/#results. - safeArgMax = windowsArgMax - maxEnvSize - ) - chunks, err := splitIntoChunks(patterns, safeArgMax) - if err != nil { - return nil, false, err - } - - if driver := findExternalDriver(cfg); driver != nil { - response, err := callDriverOnChunksEx(dedup, driver, cfg, chunks) - if err != nil { - return nil, false, err - } else if !response.NotHandled { - return response, true, nil - } - // (fall through) - } - - // go list fallback - // - // Write overlays once, as there are many calls - // to 'go list' (one per chunk plus others too). - overlay, cleanupOverlay, err := writeOverlays(cfg.Overlay) - if err != nil { - return nil, false, err - } - defer cleanupOverlay() - setGoListOverlayFile(cfg, overlay) - - response, err := callDriverOnChunksEx(dedup, goListDriver, cfg, chunks) - if err != nil { - return nil, false, err - } - return response, false, err -} +//go:linkname defaultDriver golang.org/x/tools/go/packages.defaultDriver +func defaultDriver(cfg *Config, patterns ...string) (*packages.DriverResponse, bool, error) //go:linkname newLoader golang.org/x/tools/go/packages.newLoader func newLoader(cfg *Config) *loader @@ -230,7 +110,7 @@ func refine(ld *loader, response *packages.DriverResponse) ([]*Package, error) // provided for convenient display of all errors. func LoadEx(dedup *Deduper, sizes func(types.Sizes) types.Sizes, cfg *Config, patterns ...string) ([]*Package, error) { ld := newLoader(cfg) - response, external, err := defaultDriverEx(dedup, &ld.Config, patterns...) + response, external, err := defaultDriver(&ld.Config, patterns...) if err != nil { return nil, err } From 66141071928b8f0b8368f423ea0c23052ad9beee Mon Sep 17 00:00:00 2001 From: xushiwei Date: Sat, 15 Jun 2024 22:34:31 +0800 Subject: [PATCH 3/6] refineEx --- internal/packages/load.go | 245 +++++++++++++++++++++++++++++++++++++- 1 file changed, 241 insertions(+), 4 deletions(-) diff --git a/internal/packages/load.go b/internal/packages/load.go index 49b81d8f..94b5757e 100644 --- a/internal/packages/load.go +++ b/internal/packages/load.go @@ -45,6 +45,8 @@ const ( NeedModule = packages.NeedModule NeedExportFile = packages.NeedExportFile + NeedEmbedFiles = packages.NeedEmbedFiles + NeedEmbedPatterns = packages.NeedEmbedPatterns NeedCompiledGoFiles = packages.NeedCompiledGoFiles NeedTypes = packages.NeedTypes @@ -60,9 +62,21 @@ type Config = packages.Config // A Package describes a loaded Go package. type Package = packages.Package +// loaderPackage augments Package with state used during the loading phase +type loaderPackage struct { + *Package + importErrors map[string]error // maps each bad import to its error + loadOnce sync.Once + color uint8 // for cycle detection + needsrc bool // load from source (Mode >= LoadTypes) + needtypes bool // type information is either requested or depended on + initial bool // package was matched by a pattern + goVersion int // minor version number of go command on PATH +} + // loader holds the working state of a single call to load. type loader struct { - pkgs map[string]unsafe.Pointer + pkgs map[string]*loaderPackage Config sizes types.Sizes // TODO(xsw): ensure offset of sizes parseCache map[string]unsafe.Pointer @@ -91,8 +105,231 @@ func defaultDriver(cfg *Config, patterns ...string) (*packages.DriverResponse, b //go:linkname newLoader golang.org/x/tools/go/packages.newLoader func newLoader(cfg *Config) *loader -//go:linkname refine golang.org/x/tools/go/packages.(*loader).refine -func refine(ld *loader, response *packages.DriverResponse) ([]*Package, error) +//go:linkname loadPackage golang.org/x/tools/go/packages.(*loader).loadPackage +func loadPackage(ld *loader, lpkg *loaderPackage) + +func loadRecursiveEx(ld *loader, lpkg *loaderPackage) { + lpkg.loadOnce.Do(func() { + // Load the direct dependencies, in parallel. + var wg sync.WaitGroup + for _, ipkg := range lpkg.Imports { + imp := ld.pkgs[ipkg.ID] + wg.Add(1) + go func(imp *loaderPackage) { + loadRecursiveEx(ld, imp) + wg.Done() + }(imp) + } + wg.Wait() + loadPackage(ld, lpkg) + }) +} + +func refineEx(ld *loader, response *packages.DriverResponse) ([]*Package, error) { + roots := response.Roots + rootMap := make(map[string]int, len(roots)) + for i, root := range roots { + rootMap[root] = i + } + ld.pkgs = make(map[string]*loaderPackage) + // first pass, fixup and build the map and roots + var initial = make([]*loaderPackage, len(roots)) + for _, pkg := range response.Packages { + rootIndex := -1 + if i, found := rootMap[pkg.ID]; found { + rootIndex = i + } + + // Overlays can invalidate export data. + // TODO(matloob): make this check fine-grained based on dependencies on overlaid files + exportDataInvalid := len(ld.Overlay) > 0 || pkg.ExportFile == "" && pkg.PkgPath != "unsafe" + // This package needs type information if the caller requested types and the package is + // either a root, or it's a non-root and the user requested dependencies ... + needtypes := (ld.Mode&NeedTypes|NeedTypesInfo != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0)) + // This package needs source if the call requested source (or types info, which implies source) + // and the package is either a root, or itas a non- root and the user requested dependencies... + needsrc := ((ld.Mode&(NeedSyntax|NeedTypesInfo) != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0)) || + // ... or if we need types and the exportData is invalid. We fall back to (incompletely) + // typechecking packages from source if they fail to compile. + (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && exportDataInvalid)) && pkg.PkgPath != "unsafe" + lpkg := &loaderPackage{ + Package: pkg, + needtypes: needtypes, + needsrc: needsrc, + goVersion: response.GoVersion, + } + ld.pkgs[lpkg.ID] = lpkg + if rootIndex >= 0 { + initial[rootIndex] = lpkg + lpkg.initial = true + } + } + for i, root := range roots { + if initial[i] == nil { + return nil, fmt.Errorf("root package %v is missing", root) + } + } + + if ld.Mode&NeedImports != 0 { + // Materialize the import graph. + + const ( + white = 0 // new + grey = 1 // in progress + black = 2 // complete + ) + + // visit traverses the import graph, depth-first, + // and materializes the graph as Packages.Imports. + // + // Valid imports are saved in the Packages.Import map. + // Invalid imports (cycles and missing nodes) are saved in the importErrors map. + // Thus, even in the presence of both kinds of errors, + // the Import graph remains a DAG. + // + // visit returns whether the package needs src or has a transitive + // dependency on a package that does. These are the only packages + // for which we load source code. + var stack []*loaderPackage + var visit func(lpkg *loaderPackage) bool + visit = func(lpkg *loaderPackage) bool { + switch lpkg.color { + case black: + return lpkg.needsrc + case grey: + panic("internal error: grey node") + } + lpkg.color = grey + stack = append(stack, lpkg) // push + stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports + lpkg.Imports = make(map[string]*Package, len(stubs)) + for importPath, ipkg := range stubs { + var importErr error + imp := ld.pkgs[ipkg.ID] + if imp == nil { + // (includes package "C" when DisableCgo) + importErr = fmt.Errorf("missing package: %q", ipkg.ID) + } else if imp.color == grey { + importErr = fmt.Errorf("import cycle: %s", stack) + } + if importErr != nil { + if lpkg.importErrors == nil { + lpkg.importErrors = make(map[string]error) + } + lpkg.importErrors[importPath] = importErr + continue + } + + if visit(imp) { + lpkg.needsrc = true + } + lpkg.Imports[importPath] = imp.Package + } + + // Complete type information is required for the + // immediate dependencies of each source package. + if lpkg.needsrc && ld.Mode&NeedTypes != 0 { + for _, ipkg := range lpkg.Imports { + ld.pkgs[ipkg.ID].needtypes = true + } + } + + // NeedTypeSizes causes TypeSizes to be set even + // on packages for which types aren't needed. + if ld.Mode&NeedTypesSizes != 0 { + lpkg.TypesSizes = ld.sizes + } + stack = stack[:len(stack)-1] // pop + lpkg.color = black + + return lpkg.needsrc + } + + // For each initial package, create its import DAG. + for _, lpkg := range initial { + visit(lpkg) + } + + } else { + // !NeedImports: drop the stub (ID-only) import packages + // that we are not even going to try to resolve. + for _, lpkg := range initial { + lpkg.Imports = nil + } + } + + // Load type data and syntax if needed, starting at + // the initial packages (roots of the import DAG). + if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 { + var wg sync.WaitGroup + for _, lpkg := range initial { + wg.Add(1) + go func(lpkg *loaderPackage) { + loadRecursiveEx(ld, lpkg) + wg.Done() + }(lpkg) + } + wg.Wait() + } + + // If the context is done, return its error and + // throw out [likely] incomplete packages. + if err := ld.Context.Err(); err != nil { + return nil, err + } + + result := make([]*Package, len(initial)) + for i, lpkg := range initial { + result[i] = lpkg.Package + } + for i := range ld.pkgs { + // Clear all unrequested fields, + // to catch programs that use more than they request. + if ld.requestedMode&NeedName == 0 { + ld.pkgs[i].Name = "" + ld.pkgs[i].PkgPath = "" + } + if ld.requestedMode&NeedFiles == 0 { + ld.pkgs[i].GoFiles = nil + ld.pkgs[i].OtherFiles = nil + ld.pkgs[i].IgnoredFiles = nil + } + if ld.requestedMode&NeedEmbedFiles == 0 { + ld.pkgs[i].EmbedFiles = nil + } + if ld.requestedMode&NeedEmbedPatterns == 0 { + ld.pkgs[i].EmbedPatterns = nil + } + if ld.requestedMode&NeedCompiledGoFiles == 0 { + ld.pkgs[i].CompiledGoFiles = nil + } + if ld.requestedMode&NeedImports == 0 { + ld.pkgs[i].Imports = nil + } + if ld.requestedMode&NeedExportFile == 0 { + ld.pkgs[i].ExportFile = "" + } + if ld.requestedMode&NeedTypes == 0 { + ld.pkgs[i].Types = nil + ld.pkgs[i].Fset = nil + ld.pkgs[i].IllTyped = false + } + if ld.requestedMode&NeedSyntax == 0 { + ld.pkgs[i].Syntax = nil + } + if ld.requestedMode&NeedTypesInfo == 0 { + ld.pkgs[i].TypesInfo = nil + } + if ld.requestedMode&NeedTypesSizes == 0 { + ld.pkgs[i].TypesSizes = nil + } + if ld.requestedMode&NeedModule == 0 { + ld.pkgs[i].Module = nil + } + } + + return result, nil +} // LoadEx loads and returns the Go packages named by the given patterns. // @@ -137,7 +374,7 @@ func LoadEx(dedup *Deduper, sizes func(types.Sizes) types.Sizes, cfg *Config, pa if sizes != nil { ld.sizes = sizes(ld.sizes) } - return refine(ld, response) + return refineEx(ld, response) } // Visit visits all the packages in the import graph whose roots are From 4af872ddd59f357f0d09ba0750ac4c17d197aa5c Mon Sep 17 00:00:00 2001 From: xushiwei Date: Sat, 15 Jun 2024 22:56:17 +0800 Subject: [PATCH 4/6] loadPackageEx --- internal/packages/load.go | 302 +++++++++++++++++++++++++++++++++++++- 1 file changed, 299 insertions(+), 3 deletions(-) diff --git a/internal/packages/load.go b/internal/packages/load.go index 94b5757e..f65adcc0 100644 --- a/internal/packages/load.go +++ b/internal/packages/load.go @@ -18,8 +18,13 @@ package packages import ( "fmt" + "go/ast" + "go/scanner" "go/types" + "log" + "os" "runtime" + "strings" "sync" "unsafe" @@ -52,6 +57,8 @@ const ( NeedTypes = packages.NeedTypes NeedTypesSizes = packages.NeedTypesSizes NeedTypesInfo = packages.NeedTypesInfo + + typecheckCgo = NeedModule - 1 // TODO(xsw): how to check ) // A Config specifies details about how packages should be loaded. @@ -105,8 +112,297 @@ func defaultDriver(cfg *Config, patterns ...string) (*packages.DriverResponse, b //go:linkname newLoader golang.org/x/tools/go/packages.newLoader func newLoader(cfg *Config) *loader -//go:linkname loadPackage golang.org/x/tools/go/packages.(*loader).loadPackage -func loadPackage(ld *loader, lpkg *loaderPackage) +//go:linkname loadFromExportData golang.org/x/tools/go/packages.(*loader).loadFromExportData +func loadFromExportData(ld *loader, lpkg *loaderPackage) error + +//go:linkname parseFiles golang.org/x/tools/go/packages.(*loader).parseFiles +func parseFiles(ld *loader, filenames []string) ([]*ast.File, []error) + +//go:linkname versionsInitFileVersions golang.org/x/tools/internal/versions.InitFileVersions +func versionsInitFileVersions(*types.Info) + +//go:linkname typesinternalSetUsesCgo golang.org/x/tools/internal/typesinternal.SetUsesCgo +func typesinternalSetUsesCgo(conf *types.Config) bool + +// An importFunc is an implementation of the single-method +// types.Importer interface based on a function value. +type importerFunc func(path string) (*types.Package, error) + +func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } + +func loadPackageEx(ld *loader, lpkg *loaderPackage) { + if lpkg.PkgPath == "unsafe" { + // Fill in the blanks to avoid surprises. + lpkg.Types = types.Unsafe + lpkg.Fset = ld.Fset + lpkg.Syntax = []*ast.File{} + lpkg.TypesInfo = new(types.Info) + lpkg.TypesSizes = ld.sizes + return + } + + // Call NewPackage directly with explicit name. + // This avoids skew between golist and go/types when the files' + // package declarations are inconsistent. + lpkg.Types = types.NewPackage(lpkg.PkgPath, lpkg.Name) + lpkg.Fset = ld.Fset + + // Start shutting down if the context is done and do not load + // source or export data files. + // Packages that import this one will have ld.Context.Err() != nil. + // ld.Context.Err() will be returned later by refine. + if ld.Context.Err() != nil { + return + } + + // Subtle: we populate all Types fields with an empty Package + // before loading export data so that export data processing + // never has to create a types.Package for an indirect dependency, + // which would then require that such created packages be explicitly + // inserted back into the Import graph as a final step after export data loading. + // (Hence this return is after the Types assignment.) + // The Diamond test exercises this case. + if !lpkg.needtypes && !lpkg.needsrc { + return + } + if !lpkg.needsrc { + if err := loadFromExportData(ld, lpkg); err != nil { + lpkg.Errors = append(lpkg.Errors, packages.Error{ + Pos: "-", + Msg: err.Error(), + Kind: packages.UnknownError, // e.g. can't find/open/parse export data + }) + } + return // not a source package, don't get syntax trees + } + + appendError := func(err error) { + // Convert various error types into the one true Error. + var errs []packages.Error + switch err := err.(type) { + case packages.Error: + // from driver + errs = append(errs, err) + + case *os.PathError: + // from parser + errs = append(errs, packages.Error{ + Pos: err.Path + ":1", + Msg: err.Err.Error(), + Kind: packages.ParseError, + }) + + case scanner.ErrorList: + // from parser + for _, err := range err { + errs = append(errs, packages.Error{ + Pos: err.Pos.String(), + Msg: err.Msg, + Kind: packages.ParseError, + }) + } + + case types.Error: + // from type checker + lpkg.TypeErrors = append(lpkg.TypeErrors, err) + errs = append(errs, packages.Error{ + Pos: err.Fset.Position(err.Pos).String(), + Msg: err.Msg, + Kind: packages.TypeError, + }) + + default: + // unexpected impoverished error from parser? + errs = append(errs, packages.Error{ + Pos: "-", + Msg: err.Error(), + Kind: packages.UnknownError, + }) + + // If you see this error message, please file a bug. + log.Printf("internal error: error %q (%T) without position", err, err) + } + + lpkg.Errors = append(lpkg.Errors, errs...) + } + + // If the go command on the PATH is newer than the runtime, + // then the go/{scanner,ast,parser,types} packages from the + // standard library may be unable to process the files + // selected by go list. + // + // There is currently no way to downgrade the effective + // version of the go command (see issue 52078), so we proceed + // with the newer go command but, in case of parse or type + // errors, we emit an additional diagnostic. + // + // See: + // - golang.org/issue/52078 (flag to set release tags) + // - golang.org/issue/50825 (gopls legacy version support) + // - golang.org/issue/55883 (go/packages confusing error) + // + // Should we assert a hard minimum of (currently) go1.16 here? + var runtimeVersion int + if _, err := fmt.Sscanf(runtime.Version(), "go1.%d", &runtimeVersion); err == nil && runtimeVersion < lpkg.goVersion { + defer func() { + if len(lpkg.Errors) > 0 { + appendError(packages.Error{ + Pos: "-", + Msg: fmt.Sprintf("This application uses version go1.%d of the source-processing packages but runs version go1.%d of 'go list'. It may fail to process source files that rely on newer language features. If so, rebuild the application using a newer version of Go.", runtimeVersion, lpkg.goVersion), + Kind: packages.UnknownError, + }) + } + }() + } + + if ld.Config.Mode&NeedTypes != 0 && len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" { + // The config requested loading sources and types, but sources are missing. + // Add an error to the package and fall back to loading from export data. + appendError(packages.Error{ + Pos: "-", + Msg: fmt.Sprintf("sources missing for package %s", lpkg.ID), + Kind: packages.ParseError, + }) + _ = loadFromExportData(ld, lpkg) // ignore any secondary errors + + return // can't get syntax trees for this package + } + + files, errs := parseFiles(ld, lpkg.CompiledGoFiles) + for _, err := range errs { + appendError(err) + } + + lpkg.Syntax = files + if ld.Config.Mode&NeedTypes == 0 { + return + } + + // Start shutting down if the context is done and do not type check. + // Packages that import this one will have ld.Context.Err() != nil. + // ld.Context.Err() will be returned later by refine. + if ld.Context.Err() != nil { + return + } + + lpkg.TypesInfo = &types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + Implicits: make(map[ast.Node]types.Object), + Instances: make(map[*ast.Ident]types.Instance), + Scopes: make(map[ast.Node]*types.Scope), + Selections: make(map[*ast.SelectorExpr]*types.Selection), + } + versionsInitFileVersions(lpkg.TypesInfo) + lpkg.TypesSizes = ld.sizes + + importer := importerFunc(func(path string) (*types.Package, error) { + if path == "unsafe" { + return types.Unsafe, nil + } + + // The imports map is keyed by import path. + ipkg := lpkg.Imports[path] + if ipkg == nil { + if err := lpkg.importErrors[path]; err != nil { + return nil, err + } + // There was skew between the metadata and the + // import declarations, likely due to an edit + // race, or because the ParseFile feature was + // used to supply alternative file contents. + return nil, fmt.Errorf("no metadata for %s", path) + } + + if ipkg.Types != nil && ipkg.Types.Complete() { + return ipkg.Types, nil + } + log.Fatalf("internal error: package %q without types was imported from %q", path, lpkg) + panic("unreachable") + }) + + // type-check + tc := &types.Config{ + Importer: importer, + + // Type-check bodies of functions only in initial packages. + // Example: for import graph A->B->C and initial packages {A,C}, + // we can ignore function bodies in B. + IgnoreFuncBodies: ld.Mode&NeedDeps == 0 && !lpkg.initial, + + Error: appendError, + Sizes: ld.sizes, // may be nil + } + if lpkg.Module != nil && lpkg.Module.GoVersion != "" { + tc.GoVersion = "go" + lpkg.Module.GoVersion + } + if (ld.Mode & typecheckCgo) != 0 { + if !typesinternalSetUsesCgo(tc) { + appendError(packages.Error{ + Msg: "typecheckCgo requires Go 1.15+", + Kind: packages.ListError, + }) + return + } + } + + typErr := types.NewChecker(tc, ld.Fset, lpkg.Types, lpkg.TypesInfo).Files(lpkg.Syntax) + lpkg.importErrors = nil // no longer needed + + // In go/types go1.21 and go1.22, Checker.Files failed fast with a + // a "too new" error, without calling tc.Error and without + // proceeding to type-check the package (#66525). + // We rely on the runtimeVersion error to give the suggested remedy. + if typErr != nil && len(lpkg.Errors) == 0 && len(lpkg.Syntax) > 0 { + if msg := typErr.Error(); strings.HasPrefix(msg, "package requires newer Go version") { + appendError(types.Error{ + Fset: ld.Fset, + Pos: lpkg.Syntax[0].Package, + Msg: msg, + }) + } + } + + // If !Cgo, the type-checker uses FakeImportC mode, so + // it doesn't invoke the importer for import "C", + // nor report an error for the import, + // or for any undefined C.f reference. + // We must detect this explicitly and correctly + // mark the package as IllTyped (by reporting an error). + // TODO(adonovan): if these errors are annoying, + // we could just set IllTyped quietly. + if tc.FakeImportC { + outer: + for _, f := range lpkg.Syntax { + for _, imp := range f.Imports { + if imp.Path.Value == `"C"` { + err := types.Error{Fset: ld.Fset, Pos: imp.Pos(), Msg: `import "C" ignored`} + appendError(err) + break outer + } + } + } + } + + // If types.Checker.Files had an error that was unreported, + // make sure to report the unknown error so the package is illTyped. + if typErr != nil && len(lpkg.Errors) == 0 { + appendError(typErr) + } + + // Record accumulated errors. + illTyped := len(lpkg.Errors) > 0 + if !illTyped { + for _, imp := range lpkg.Imports { + if imp.IllTyped { + illTyped = true + break + } + } + } + lpkg.IllTyped = illTyped +} func loadRecursiveEx(ld *loader, lpkg *loaderPackage) { lpkg.loadOnce.Do(func() { @@ -121,7 +417,7 @@ func loadRecursiveEx(ld *loader, lpkg *loaderPackage) { }(imp) } wg.Wait() - loadPackage(ld, lpkg) + loadPackageEx(ld, lpkg) }) } From 1b48b98e22874464c28c2fd4d3af1694772da8e5 Mon Sep 17 00:00:00 2001 From: xushiwei Date: Sat, 15 Jun 2024 23:03:29 +0800 Subject: [PATCH 5/6] refineEx: Deduper --- internal/packages/load.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/packages/load.go b/internal/packages/load.go index f65adcc0..92ba808a 100644 --- a/internal/packages/load.go +++ b/internal/packages/load.go @@ -130,7 +130,7 @@ type importerFunc func(path string) (*types.Package, error) func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } -func loadPackageEx(ld *loader, lpkg *loaderPackage) { +func loadPackageEx(dedup *Deduper, ld *loader, lpkg *loaderPackage) { if lpkg.PkgPath == "unsafe" { // Fill in the blanks to avoid surprises. lpkg.Types = types.Unsafe @@ -404,7 +404,7 @@ func loadPackageEx(ld *loader, lpkg *loaderPackage) { lpkg.IllTyped = illTyped } -func loadRecursiveEx(ld *loader, lpkg *loaderPackage) { +func loadRecursiveEx(dedup *Deduper, ld *loader, lpkg *loaderPackage) { lpkg.loadOnce.Do(func() { // Load the direct dependencies, in parallel. var wg sync.WaitGroup @@ -412,16 +412,16 @@ func loadRecursiveEx(ld *loader, lpkg *loaderPackage) { imp := ld.pkgs[ipkg.ID] wg.Add(1) go func(imp *loaderPackage) { - loadRecursiveEx(ld, imp) + loadRecursiveEx(dedup, ld, imp) wg.Done() }(imp) } wg.Wait() - loadPackageEx(ld, lpkg) + loadPackageEx(dedup, ld, lpkg) }) } -func refineEx(ld *loader, response *packages.DriverResponse) ([]*Package, error) { +func refineEx(dedup *Deduper, ld *loader, response *packages.DriverResponse) ([]*Package, error) { roots := response.Roots rootMap := make(map[string]int, len(roots)) for i, root := range roots { @@ -561,7 +561,7 @@ func refineEx(ld *loader, response *packages.DriverResponse) ([]*Package, error) for _, lpkg := range initial { wg.Add(1) go func(lpkg *loaderPackage) { - loadRecursiveEx(ld, lpkg) + loadRecursiveEx(dedup, ld, lpkg) wg.Done() }(lpkg) } @@ -670,7 +670,7 @@ func LoadEx(dedup *Deduper, sizes func(types.Sizes) types.Sizes, cfg *Config, pa if sizes != nil { ld.sizes = sizes(ld.sizes) } - return refineEx(ld, response) + return refineEx(dedup, ld, response) } // Visit visits all the packages in the import graph whose roots are From 2b1da5b2312e132844d1ea24b9254534aae53ca1 Mon Sep 17 00:00:00 2001 From: xushiwei Date: Sat, 15 Jun 2024 23:27:41 +0800 Subject: [PATCH 6/6] loadPackageEx: dedup --- cl/compile_test.go | 13 ++++------ internal/packages/load.go | 52 ++++++++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/cl/compile_test.go b/cl/compile_test.go index 8c4f9706..ad15268c 100644 --- a/cl/compile_test.go +++ b/cl/compile_test.go @@ -19,7 +19,6 @@ package cl_test import ( "os" "path/filepath" - "runtime" "testing" "github.com/goplus/llgo/cl" @@ -71,13 +70,11 @@ func TestPython(t *testing.T) { cltest.Pkg(t, ssa.PkgPython, "../py/llgo_autogen.ll") } -func TestGoLibMath(t *testing.T) { - if runtime.GOOS == "darwin" { // TODO(xsw): support linux/windows - root, _ := filepath.Abs("..") - os.Setenv("LLGOROOT", root) - conf := build.NewDefaultConf(build.ModeInstall) - build.Do([]string{"math"}, conf) - } +func TestGoPkgMath(t *testing.T) { + root, _ := filepath.Abs("..") + os.Setenv("LLGOROOT", root) + conf := build.NewDefaultConf(build.ModeInstall) + build.Do([]string{"math"}, conf) } func TestVar(t *testing.T) { diff --git a/internal/packages/load.go b/internal/packages/load.go index 92ba808a..8f8640eb 100644 --- a/internal/packages/load.go +++ b/internal/packages/load.go @@ -99,13 +99,33 @@ type loader struct { requestedMode LoadMode } -type Deduper struct { +type cachedPackage struct { + Types *types.Package + TypesInfo *types.Info + Syntax []*ast.File } -func NewDeduper() *Deduper { +type aDeduper struct { + cache sync.Map +} + +type Deduper = *aDeduper + +func NewDeduper() Deduper { + return &aDeduper{} +} + +func (p Deduper) check(pkgPath string) *cachedPackage { + if v, ok := p.cache.Load(pkgPath); ok { + return v.(*cachedPackage) + } return nil } +func (p Deduper) set(pkgPath string, cp *cachedPackage) { + p.cache.Store(pkgPath, cp) +} + //go:linkname defaultDriver golang.org/x/tools/go/packages.defaultDriver func defaultDriver(cfg *Config, patterns ...string) (*packages.DriverResponse, bool, error) @@ -130,7 +150,7 @@ type importerFunc func(path string) (*types.Package, error) func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } -func loadPackageEx(dedup *Deduper, ld *loader, lpkg *loaderPackage) { +func loadPackageEx(dedup Deduper, ld *loader, lpkg *loaderPackage) { if lpkg.PkgPath == "unsafe" { // Fill in the blanks to avoid surprises. lpkg.Types = types.Unsafe @@ -141,6 +161,26 @@ func loadPackageEx(dedup *Deduper, ld *loader, lpkg *loaderPackage) { return } + if dedup != nil { + if cp := dedup.check(lpkg.PkgPath); cp != nil { + lpkg.Types = cp.Types + lpkg.Fset = ld.Fset + lpkg.TypesInfo = cp.TypesInfo + lpkg.Syntax = cp.Syntax + lpkg.TypesSizes = ld.sizes + return + } + defer func() { + if !lpkg.IllTyped && lpkg.needtypes && lpkg.needsrc { + dedup.set(lpkg.PkgPath, &cachedPackage{ + Types: lpkg.Types, + TypesInfo: lpkg.TypesInfo, + Syntax: lpkg.Syntax, + }) + } + }() + } + // Call NewPackage directly with explicit name. // This avoids skew between golist and go/types when the files' // package declarations are inconsistent. @@ -404,7 +444,7 @@ func loadPackageEx(dedup *Deduper, ld *loader, lpkg *loaderPackage) { lpkg.IllTyped = illTyped } -func loadRecursiveEx(dedup *Deduper, ld *loader, lpkg *loaderPackage) { +func loadRecursiveEx(dedup Deduper, ld *loader, lpkg *loaderPackage) { lpkg.loadOnce.Do(func() { // Load the direct dependencies, in parallel. var wg sync.WaitGroup @@ -421,7 +461,7 @@ func loadRecursiveEx(dedup *Deduper, ld *loader, lpkg *loaderPackage) { }) } -func refineEx(dedup *Deduper, ld *loader, response *packages.DriverResponse) ([]*Package, error) { +func refineEx(dedup Deduper, ld *loader, response *packages.DriverResponse) ([]*Package, error) { roots := response.Roots rootMap := make(map[string]int, len(roots)) for i, root := range roots { @@ -641,7 +681,7 @@ func refineEx(dedup *Deduper, ld *loader, response *packages.DriverResponse) ([] // return an error. Clients may need to handle such errors before // proceeding with further analysis. The PrintErrors function is // provided for convenient display of all errors. -func LoadEx(dedup *Deduper, sizes func(types.Sizes) types.Sizes, cfg *Config, patterns ...string) ([]*Package, error) { +func LoadEx(dedup Deduper, sizes func(types.Sizes) types.Sizes, cfg *Config, patterns ...string) ([]*Package, error) { ld := newLoader(cfg) response, external, err := defaultDriver(&ld.Config, patterns...) if err != nil {