repos: flesh out package more, use it in maintnerd

This removes yet another list of all our repos.

(Goal is to simplify https://github.com/golang/go/wiki/CreatingSubRepository)

Most of the work was in earlier CL 208697.

Updates golang/go#36047

Change-Id: I9147b959fe6574e2f809091d08d360051b69402e
Reviewed-on: https://go-review.googlesource.com/c/build/+/210461
Run-TryBot: Brad Fitzpatrick <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
Reviewed-by: Alexander Rakoczy <[email protected]>

Author: bradfitz

app/appengine/dash.go

@@ -125,7 +125,7 @@ var goDash = &Dashboard{
 func init() {
 	var add []*Package
 	for _, r := range repos.ByGerritProject {
-		if r.HideFromDashboard || !strings.HasPrefix(r.ImportPath, "golang.org/x") || r.GoGerritProject == "" {
+		if !r.ShowOnDashboard() {
 			continue
 		}
 		add = append(add, &Package{

app/appengine/ui.go

@@ -245,7 +245,7 @@ func repoImportPath(rh *apipb.DashRepoHead) string {
 		return ""
 	}
 	ri, ok := repos.ByGerritProject[rh.GerritProject]
-	if !ok || ri.HideFromDashboard {
+	if !ok || !ri.ShowOnDashboard() {
 		return ""
 	}
 	return ri.ImportPath

cmd/gitmirror/gitmirror.go

@@ -188,11 +188,8 @@ func handleRoot(w http.ResponseWriter, r *http.Request) {
 // shouldMirrorTo returns the GitHub repository the named repo should be
 // mirrored to or "" if it should not be mirrored.
 func shouldMirrorTo(name string) (dst string) {
-	if name == "protobuf" {
-		return "[email protected]:protocolbuffers/protobuf-go.git"
-	}
-	if r, ok := repospkg.ByGerritProject[name]; ok && r.MirroredToGithub {
-		return "[email protected]:golang/" + name + ".git"
+	if r, ok := repospkg.ByGerritProject[name]; ok && r.MirrorToGitHub {
+		return "[email protected]:" + r.GitHubRepo() + ".git"
 	}
 	return ""
 }

maintner/maintnerd/maintnerd.go

@@ -7,7 +7,6 @@
 package main
 
 import (
-	"bytes"
 	"context"
 	"crypto/tls"
 	"flag"
@@ -22,20 +21,21 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
 
 	"cloud.google.com/go/compute/metadata"
 	"cloud.google.com/go/storage"
 	"golang.org/x/build/autocertcache"
-	"golang.org/x/build/gerrit"
 	"golang.org/x/build/internal/gitauth"
 	"golang.org/x/build/maintner"
 	"golang.org/x/build/maintner/godata"
 	"golang.org/x/build/maintner/maintnerd/apipb"
 	"golang.org/x/build/maintner/maintnerd/gcslog"
 	"golang.org/x/build/maintner/maintnerd/maintapi"
+	"golang.org/x/build/repos"
 	"golang.org/x/crypto/acme/autocert"
 	"golang.org/x/net/http2"
 	"golang.org/x/time/rate"
@@ -306,46 +306,6 @@ func main() {
 	log.Fatal(<-errc)
 }
 
-// Projects to watch when using the "go" config.
-var goGitHubProjects = []string{
-	"golang/arch",
-	"golang/benchmarks",
-	"golang/blog",
-	"golang/build",
-	"golang/crypto",
-	"golang/debug",
-	"golang/dl",
-	"golang/example",
-	"golang/exp",
-	"golang/gddo",
-	"golang/go",
-	"golang/image",
-	"golang/lint",
-	"golang/mobile",
-	"golang/mod",
-	"golang/net",
-	"golang/oauth2",
-	"golang/perf",
-	"golang/playground",
-	"golang/proposal",
-	"golang/review",
-	"golang/scratch",
-	"golang/sublime-build",
-	"golang/sublime-config",
-	"golang/sync",
-	"golang/sys",
-	"golang/talks",
-	"golang/term",
-	"golang/text",
-	"golang/time",
-	"golang/tools",
-	"golang/tour",
-	"golang/vgo",
-	"golang/website",
-	"golang/xerrors",
-	"protocolbuffers/protobuf-go",
-}
-
 func setGoConfig() {
 	if *watchGithub != "" {
 		log.Fatalf("can't set both --config and --watch-github")
@@ -354,20 +314,42 @@ func setGoConfig() {
 		log.Fatalf("can't set both --config and --watch-gerrit")
 	}
 	*pubsub = "https://pubsubhelper.golang.org"
-	*watchGithub = strings.Join(goGitHubProjects, ",")
+	*watchGithub = strings.Join(goGitHubProjects(), ",")
+	*watchGerrit = strings.Join(goGerritProjects(), ",")
+}
 
-	gerrc := gerrit.NewClient("https://go-review.googlesource.com", gerrit.NoAuth)
-	projs, err := gerrc.ListProjects(context.Background())
-	if err != nil {
-		log.Fatalf("error listing Go's gerrit projects: %v", err)
-	}
-	var buf bytes.Buffer
-	buf.WriteString("code.googlesource.com/gocloud,code.googlesource.com/google-api-go-client")
-	for _, pi := range projs {
-		buf.WriteString(",go.googlesource.com/")
-		buf.WriteString(pi.ID)
+// goGitHubProjects returns the GitHub repos to track in --config=go.
+// The strings are of form "<org-or-user>/<repo>".
+func goGitHubProjects() []string {
+	var ret []string
+	for _, r := range repos.ByGerritProject {
+		if gr := r.GitHubRepo(); gr != "" {
+			ret = append(ret, gr)
+		}
 	}
-	*watchGerrit = buf.String()
+	sort.Strings(ret)
+	return ret
+}
+
+// goGerritProjects returns the Gerrit projects to track in --config=go.
+// The strings are of the form "<hostname>/<proj>".
+func goGerritProjects() []string {
+	var ret []string
+	// TODO: add these to the repos package at some point? Or
+	// maybe just stop maintaining them in maintner if nothing's
+	// using them? I think the only thing that uses them is the
+	// stats tooling, to see where gophers are working. That's
+	// probably enough reason to keep them in. So just keep hard-coding
+	// them here for now.
+	ret = append(ret,
+		"code.googlesource.com/gocloud",
+		"code.googlesource.com/google-api-go-client",
+	)
+	for p := range repos.ByGerritProject {
+		ret = append(ret, "go.googlesource.com/"+p)
+	}
+	sort.Strings(ret)
+	return ret
 }
 
 func setGodataConfig() {

repos/repos.go

@@ -8,22 +8,31 @@ package repos
 import "fmt"
 
 type Repo struct {
-	// GoGerritProject, if non-empty, is its Gerrit project name,
-	// such as "go", "net", or "sys".
+	// GoGerritProject, if non-empty, is the repo's Gerrit project
+	// name, such as "go", "net", or "sys".
 	GoGerritProject string
 
 	// ImportPath is the repo's import path.
 	// It is empty for the main Go repo.
 	ImportPath string
 
-	MirroredToGithub bool
+	// MirrorToGitHub controls whether this repo is mirrored
+	// from Gerrit to GitHub. If true, GoGerritProject and
+	// gitHubRepo must both be defined.
+	MirrorToGitHub bool
 
-	// HideFromDashboard, if true, makes the repo not appear at build.golang.org.
-	HideFromDashboard bool
+	// showOnDashboard is whether to show the repo on the bottom
+	// of build.golang.org in the repo overview section.
+	showOnDashboard bool
 
 	// CoordinatorCanBuild reports whether this a repo that the
 	// build coordinator knows how to build.
 	CoordinatorCanBuild bool
+
+	// gitHubRepo is the "org/repo" of where this repo exists on
+	// GitHub. If MirrorToGitHub is true, this is the
+	// destination.
+	gitHubRepo string
 }
 
 // ByGerritProject maps from a Gerrit project name ("go", "net", etc)
@@ -35,13 +44,14 @@ var ByGerritProject = map[string]*Repo{ /* initialized below */ }
 var ByImportPath = map[string]*Repo{ /* initialized below */ }
 
 func init() {
-	add(&Repo{GoGerritProject: "go", MirroredToGithub: true, CoordinatorCanBuild: true})
-	add(&Repo{GoGerritProject: "dl", MirroredToGithub: true, ImportPath: "golang.org/dl", HideFromDashboard: true, CoordinatorCanBuild: true})
-	add(&Repo{GoGerritProject: "protobuf", MirroredToGithub: true, ImportPath: "github.com/google/protobuf", HideFromDashboard: true})
-	add(&Repo{GoGerritProject: "gddo", MirroredToGithub: true, ImportPath: "github.com/golang/gddo", HideFromDashboard: true})
-	add(&Repo{GoGerritProject: "gofrontend", MirroredToGithub: true, HideFromDashboard: true})
-	add(&Repo{GoGerritProject: "gollvm", MirroredToGithub: false, HideFromDashboard: true})
-	add(&Repo{GoGerritProject: "grpc-review", MirroredToGithub: false, HideFromDashboard: true})
+	addMirrored("go", coordinatorCanBuild, noDash)
+	addMirrored("dl", importPath("golang.org/dl"), coordinatorCanBuild)
+	addMirrored("gddo", importPath("github.com/golang/gddo"))
+	addMirrored("gofrontend")
+	addMirrored("proposal")
+	addMirrored("sublime-build")
+	addMirrored("sublime-config")
+
 	x("arch")
 	x("benchmarks")
 	x("blog")
@@ -71,21 +81,50 @@ func init() {
 	x("vgo", noDash)
 	x("website")
 	x("xerrors", noDash)
+
+	add(&Repo{GoGerritProject: "gollvm"})
+	add(&Repo{GoGerritProject: "grpc-review"})
+
+	add(&Repo{
+		GoGerritProject: "protobuf",
+		MirrorToGitHub:  true,
+		ImportPath:      "github.com/google/protobuf",
+		gitHubRepo:      "protocolbuffers/protobuf-go",
+	})
 }
 
 type modifyRepo func(*Repo)
 
 // noDash is an option to the x func that marks the repo as hidden on
 // the https://build.golang.org/ dashboard.
-func noDash(r *Repo) { r.HideFromDashboard = true }
+func noDash(r *Repo) { r.showOnDashboard = false }
+
+func coordinatorCanBuild(r *Repo) { r.CoordinatorCanBuild = true }
+
+func importPath(v string) modifyRepo { return func(r *Repo) { r.ImportPath = v } }
+
+// addMirrored adds a repo that's on Gerrit and mirrored to GitHub.
+func addMirrored(proj string, opts ...modifyRepo) {
+	repo := &Repo{
+		GoGerritProject: proj,
+		MirrorToGitHub:  true,
+		gitHubRepo:      "golang/" + proj,
+	}
+	for _, o := range opts {
+		o(repo)
+	}
+	add(repo)
+}
 
 // x adds a golang.org/x repo.
 func x(proj string, opts ...modifyRepo) {
 	repo := &Repo{
 		GoGerritProject:     proj,
-		MirroredToGithub:    true,
+		MirrorToGitHub:      true,
 		CoordinatorCanBuild: true,
 		ImportPath:          "golang.org/x/" + proj,
+		gitHubRepo:          "golang/" + proj,
+		showOnDashboard:     true,
 	}
 	for _, o := range opts {
 		o(repo)
@@ -94,6 +133,23 @@ func x(proj string, opts ...modifyRepo) {
 }
 
 func add(r *Repo) {
+	if r.MirrorToGitHub {
+		if r.gitHubRepo == "" {
+			panic(fmt.Sprintf("project %+v has MirrorToGitHub but no gitHubRepo", r))
+		}
+		if r.GoGerritProject == "" {
+			panic(fmt.Sprintf("project %+v has MirrorToGitHub but no GoGerritProject", r))
+		}
+	}
+	if r.showOnDashboard {
+		if !r.CoordinatorCanBuild {
+			panic(fmt.Sprintf("project %+v is showOnDashboard but not marked buildable by coordinator", r))
+		}
+		if r.GoGerritProject == "" {
+			panic(fmt.Sprintf("project %+v is showOnDashboard but has no Gerrit project", r))
+		}
+	}
+
 	if p := r.GoGerritProject; p != "" {
 		if _, dup := ByGerritProject[p]; dup {
 			panic(fmt.Sprintf("duplicate Gerrit project %q in %+v", p, r))
@@ -107,3 +163,14 @@ func add(r *Repo) {
 		ByImportPath[p] = r
 	}
 }
+
+// ShowOnDashboard reports whether this repo should show up on build.golang.org
+// in the list of repos at bottom.
+//
+// When this returns true, r.GoGerritProject is guaranteed to be non-empty.
+func (r *Repo) ShowOnDashboard() bool { return r.showOnDashboard }
+
+// GitHubRepo returns the "<org>/<repo>" that this repo either lives
+// at or is mirrored to. It returns the empty string if this repo has no
+// GitHub presence.
+func (r *Repo) GitHubRepo() string { return r.gitHubRepo }

repos/repos_test.go

@@ -0,0 +1,12 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package repos
+
+import "testing"
+
+func TestInitDoesNotPanic(t *testing.T) {
+	// Verify that repos.go's init funcs don't panic when
+	// validating the repos.
+}