From ea85bd5a6d47693a0bf86ccb62c2295204b6c392 Mon Sep 17 00:00:00 2001 From: Alex Torok Date: Mon, 26 Aug 2024 19:23:31 +0000 Subject: [PATCH] Add a lazy indexing mode to gazelle --- cmd/gazelle/fix-update.go | 45 ++++++++++++++++++++++++++++++++++++++- config/config.go | 9 +++++++- language/lang.go | 3 +++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/cmd/gazelle/fix-update.go b/cmd/gazelle/fix-update.go index a4c3b32fe..e1d1a625c 100644 --- a/cmd/gazelle/fix-update.go +++ b/cmd/gazelle/fix-update.go @@ -136,7 +136,11 @@ func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) erro uc.dirs[i] = dir } - if ucr.recursive && c.IndexLibraries { + if ucr.recursive && c.LazyIndex { + uc.walkMode = walk.UpdateSubdirsMode + } else if c.LazyIndex { + uc.walkMode = walk.UpdateDirsMode + } else if ucr.recursive && c.IndexLibraries { uc.walkMode = walk.VisitAllUpdateSubdirsMode } else if c.IndexLibraries { uc.walkMode = walk.VisitAllUpdateDirsMode @@ -240,6 +244,8 @@ type visitRecord struct { // empty is a list of empty Go rules that may be deleted. empty []*rule.Rule + toIndex map[string]struct{} + // file is the build file being processed. file *rule.File @@ -311,8 +317,11 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) { } }() + visitedPkgs := make(map[string]struct{}) + var errorsFromWalk []error walk.Walk(c, cexts, uc.dirs, uc.walkMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) { + visitedPkgs[rel] = struct{}{} // If this file is ignored or if Gazelle was not asked to update this // directory, just index the build file and move on. if !update { @@ -337,6 +346,7 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) { // Generate rules. var empty, gen []*rule.Rule var imports []interface{} + toIndex := map[string]struct{}{} for _, l := range filterLanguages(c, languages) { res := l.GenerateRules(language.GenerateArgs{ Config: c, @@ -355,6 +365,9 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) { empty = append(empty, res.Empty...) gen = append(gen, res.Gen...) imports = append(imports, res.Imports...) + for _, pkg := range res.PackagesToIndex { + toIndex[pkg] = struct{}{} + } } if f == nil && len(gen) == 0 { return @@ -439,6 +452,7 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) { c: c, rules: gen, imports: imports, + toIndex: toIndex, empty: empty, file: f, mappedKinds: mappedKinds, @@ -472,6 +486,35 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) { return fmt.Errorf("encountered multiple errors: %w, %v", errorsFromWalk[0], strings.Join(additionalErrors, ", ")) } + // Index the directories that we were asked to index + if c.LazyIndex { + allToIndex := map[string]interface{}{} + for _, v := range visits { + for pkg := range v.toIndex { + if _, ok := visitedPkgs[pkg]; !ok { + allToIndex[pkg] = struct{}{} + } + } + } + + dirsToWalk := make([]string, 0, len(allToIndex)) + for pkg := range allToIndex { + // TODO: Maybe want to do a similar EvalSymlinks logic that is done + // in CheckFlags to convert args to filepaths. + dirsToWalk = append(dirsToWalk, filepath.Join(c.WorkDir, pkg)) + } + walk.Walk(c, cexts, dirsToWalk, walk.UpdateDirsMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) { + if f != nil { + for _, repl := range c.KindMap { + mrslv.MappedKind(rel, repl) + } + for _, r := range f.Rules { + ruleIndex.AddRule(c, r, f) + } + } + }) + } + // Finish building the index for dependency resolution. ruleIndex.Finish() diff --git a/config/config.go b/config/config.go index 57c3d4481..4726e16d8 100644 --- a/config/config.go +++ b/config/config.go @@ -27,6 +27,7 @@ limitations under the License. package config import ( + "errors" "flag" "fmt" "log" @@ -85,6 +86,7 @@ type Config struct { // IndexLibraries determines whether Gazelle should build an index of // libraries in the workspace for dependency resolution IndexLibraries bool + LazyIndex bool // KindMap maps from a kind name to its replacement. It provides a way for // users to customize the kind of rules created by Gazelle, via @@ -199,7 +201,7 @@ type Configurer interface { // i.e., those that apply to Config itself and not to Config.Exts. type CommonConfigurer struct { repoRoot, buildFileNames, readBuildFilesDir, writeBuildFilesDir string - indexLibraries, strict bool + indexLibraries, strict, lazyIndex bool langCsv string bzlmod bool } @@ -208,6 +210,7 @@ func (cc *CommonConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *Confi fs.StringVar(&cc.repoRoot, "repo_root", "", "path to a directory which corresponds to go_prefix, otherwise gazelle searches for it.") fs.StringVar(&cc.buildFileNames, "build_file_name", strings.Join(DefaultValidBuildFileNames, ","), "comma-separated list of valid build file names.\nThe first element of the list is the name of output build files to generate.") fs.BoolVar(&cc.indexLibraries, "index", true, "when true, gazelle will build an index of libraries in the workspace for dependency resolution") + fs.BoolVar(&cc.lazyIndex, "lazy_index", false, "when true, gazelle will lazily index") fs.BoolVar(&cc.strict, "strict", false, "when true, gazelle will exit with none-zero value for build file syntax errors or unknown directives") fs.StringVar(&cc.readBuildFilesDir, "experimental_read_build_files_dir", "", "path to a directory where build files should be read from (instead of -repo_root)") fs.StringVar(&cc.writeBuildFilesDir, "experimental_write_build_files_dir", "", "path to a directory where build files should be written to (instead of -repo_root)") @@ -250,7 +253,11 @@ func (cc *CommonConfigurer) CheckFlags(fs *flag.FlagSet, c *Config) error { c.WriteBuildFilesDir = filepath.Join(c.WorkDir, cc.writeBuildFilesDir) } } + if cc.lazyIndex && !cc.indexLibraries { + return errors.New("Using -lazy_index requires -index=true") + } c.IndexLibraries = cc.indexLibraries + c.LazyIndex = cc.lazyIndex c.Strict = cc.strict if len(cc.langCsv) > 0 { c.Langs = strings.Split(cc.langCsv, ",") diff --git a/language/lang.go b/language/lang.go index c835f74b2..d0782f914 100644 --- a/language/lang.go +++ b/language/lang.go @@ -189,4 +189,7 @@ type GenerateResult struct { // correspond. These values are passed to Resolve after merge. The type // is opaque since different languages may use different representations. Imports []interface{} + + // Additional packages that should be indexed. + PackagesToIndex []string }