From 8ca58267e91c90c3a18f6aed4d8df6cf59a9ad07 Mon Sep 17 00:00:00 2001 From: Paco Xu Date: Mon, 9 Jan 2023 16:51:22 +0800 Subject: [PATCH] add GITHUB_TOKEN support by env --- .github/workflows/run.yml | 3 +++ github.go | 48 +++++++++++++++++++++++++++++++++++---- go.mod | 4 ++++ go.sum | 4 ++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 1fce4eb..a9e12ac 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -11,6 +11,7 @@ on: env: GITHUB_NAME: pacoxu-bot GITHUB_EMAIL: paco2023@163.com + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: build: @@ -21,6 +22,8 @@ jobs: - name: My GitHub Status uses: pacoxu/github-repos-stats@main + env: + GITHUB_TOKEN: "${{ env.GITHUB_TOKEN }}" - name: Push README run: | git config --local user.email "${{ env.GITHUB_EMAIL }}" diff --git a/github.go b/github.go index b66cc8e..4971614 100644 --- a/github.go +++ b/github.go @@ -8,6 +8,7 @@ import ( "fmt" "io/ioutil" "log" + "net/http" "os" "path" "regexp" @@ -18,6 +19,7 @@ import ( "github.com/google/go-github/v42/github" "github.com/olekukonko/tablewriter" + "golang.org/x/oauth2" ) var ( @@ -72,21 +74,47 @@ func fetchAllRepos(projectsPath string, client *github.Client) ([]*github.Reposi log.Println("repos: ", reposList) // #TODO #2 Github has a rate limit of 60 per hour, so don't add more than 60 projects now var skippedRepos []string + length := len(reposList) for repo, _ := range reposList { if len(strings.Split(repo, "/")) == 2 { splits := strings.Split(repo, "/") // org/reponame is splits repository, _, err := client.Repositories.Get(context.Background(), splits[0], splits[1]) if err != nil { - log.Printf("WARNING: skip repo: %s for error: %s", repo, err) - skippedRepos = append(skippedRepos, repo) - continue + // for API rate limit error, sleep for a while + if strings.Contains(err.Error(), "403 API rate limit") { + if client == nil { + log.Printf("WARNING: sleep 3600 seconds for unauthorized github API rate limit 60 per hour: %s ", repo) + time.Sleep(3600 * time.Second) + } else { + // TODO: to get the date after `until` may be better + log.Printf("WARNING: sleep 60 seconds for authorized github API rate limit 5000 per hour: %s ", repo) + time.Sleep(60 * time.Second) + } + } + // retry once + repository, _, err = client.Repositories.Get(context.Background(), splits[0], splits[1]) + if err != nil { + log.Printf("WARNING: skip repo: %s for error: %s", repo, err) + skippedRepos = append(skippedRepos, repo) + continue + } } allRepos = append(allRepos, repository) } else { log.Printf("WARNING: skip repo: %s for error not a github repo", repo) } + // api limit by default 60 times per hour + if client == nil && length > 60 { + log.Println("Info: sleep 60 seconds as github API rate limit is 60 per hour for unauthenized user.") + time.Sleep(60 * time.Second) + } + // api limit by default 60 times per hour + if client != nil && length > 5000 { + log.Println("Info: sleep 1 seconds as github API rate limit is 5000 per hour for authenized user.") + time.Sleep(1 * time.Second) + } } log.Println("repos: ", reposList) return allRepos, skippedRepos @@ -148,7 +176,19 @@ func makeReposString(repos []*github.Repository) string { func main() { flag.Parse() - client := github.NewClient(nil) + var client *github.Client + if token, ok := os.LookupEnv("GITHUB_TOKEN"); ok { + httpClient := &http.Client{ + Transport: &oauth2.Transport{ + Base: http.DefaultTransport, + Source: oauth2.ReuseTokenSource(nil, oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})), + }, + } + client = github.NewClient(httpClient) + } else { + client = github.NewClient(nil) + + } repos, skippedRepos := fetchAllRepos(projectsPath, client) // change sort logic here sort.Slice(repos[:], func(i, j int) bool { diff --git a/go.mod b/go.mod index 36d1030..510d7c0 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,14 @@ go 1.17 require ( github.com/google/go-github/v42 v42.0.0 github.com/olekukonko/tablewriter v0.0.5 + golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be ) require ( + github.com/golang/protobuf v1.3.2 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect + google.golang.org/appengine v1.6.7 // indirect ) diff --git a/go.sum b/go.sum index 1a38f38..98b28e2 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ github.com/bradleyfalzon/ghinstallation/v2 v2.0.3/go.mod h1:tlgi+JWCXnKFx/Y4WtnDbZEINo31N5bcvnCoqieefmk= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= @@ -18,7 +19,9 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -29,4 +32,5 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=