Merge pull request #5 from pacoxu/github-token

add GITHUB_TOKEN support by env

.github/workflows/run.yml

@@ -11,6 +11,7 @@ on:
 env:
   GITHUB_NAME: pacoxu-bot
   GITHUB_EMAIL: [email protected]
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
   build:
@@ -21,6 +22,8 @@ jobs:
 
       - name: My GitHub Status
         uses: pacoxu/github-repos-stats@main
+        env:
+          GITHUB_TOKEN: "${{ env.GITHUB_TOKEN }}"
       - name: Push README
         run: |
           git config --local user.email "${{ env.GITHUB_EMAIL }}"

github.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"path"
 	"regexp"
@@ -18,6 +19,7 @@ import (
 
 	"github.com/google/go-github/v42/github"
 	"github.com/olekukonko/tablewriter"
+	"golang.org/x/oauth2"
 )
 
 var (
@@ -72,21 +74,47 @@ func fetchAllRepos(projectsPath string, client *github.Client) ([]*github.Reposi
 	log.Println("repos: ", reposList)
 	// #TODO #2 Github has a rate limit of 60 per hour, so don't add more than 60 projects now
 	var skippedRepos []string
+	length := len(reposList)
 	for repo, _ := range reposList {
 		if len(strings.Split(repo, "/")) == 2 {
 			splits := strings.Split(repo, "/")
 			// org/reponame is splits
 			repository, _, err := client.Repositories.Get(context.Background(), splits[0], splits[1])
 			if err != nil {
-				log.Printf("WARNING: skip repo: %s for error: %s", repo, err)
-				skippedRepos = append(skippedRepos, repo)
-				continue
+				// for API rate limit error, sleep for a while
+				if strings.Contains(err.Error(), "403 API rate limit") {
+					if client == nil {
+						log.Printf("WARNING: sleep 3600 seconds for unauthorized github API rate limit 60 per hour: %s ", repo)
+						time.Sleep(3600 * time.Second)
+					} else {
+						// TODO: to get the date after `until` may be better
+						log.Printf("WARNING: sleep 60 seconds for authorized github API rate limit 5000 per hour: %s ", repo)
+						time.Sleep(60 * time.Second)
+					}
+				}
+				// retry once
+				repository, _, err = client.Repositories.Get(context.Background(), splits[0], splits[1])
+				if err != nil {
+					log.Printf("WARNING: skip repo: %s for error: %s", repo, err)
+					skippedRepos = append(skippedRepos, repo)
+					continue
+				}
 			}
 			allRepos = append(allRepos, repository)
 		} else {
 			log.Printf("WARNING: skip repo: %s for error not a github repo", repo)
 		}
 
+		// api limit by default 60 times per hour
+		if client == nil && length > 60 {
+			log.Println("Info: sleep 60 seconds as github API rate limit is 60 per hour for unauthenized user.")
+			time.Sleep(60 * time.Second)
+		}
+		// api limit by default 60 times per hour
+		if client != nil && length > 5000 {
+			log.Println("Info: sleep 1 seconds as github API rate limit is 5000 per hour for authenized user.")
+			time.Sleep(1 * time.Second)
+		}
 	}
 	log.Println("repos: ", reposList)
 	return allRepos, skippedRepos
@@ -148,7 +176,19 @@ func makeReposString(repos []*github.Repository) string {
 
 func main() {
 	flag.Parse()
-	client := github.NewClient(nil)
+	var client *github.Client
+	if token, ok := os.LookupEnv("GITHUB_TOKEN"); ok {
+		httpClient := &http.Client{
+			Transport: &oauth2.Transport{
+				Base:   http.DefaultTransport,
+				Source: oauth2.ReuseTokenSource(nil, oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})),
+			},
+		}
+		client = github.NewClient(httpClient)
+	} else {
+		client = github.NewClient(nil)
+
+	}
 	repos, skippedRepos := fetchAllRepos(projectsPath, client)
 	// change sort logic here
 	sort.Slice(repos[:], func(i, j int) bool {

go.mod

@@ -5,10 +5,14 @@ go 1.17
 require (
 	github.com/google/go-github/v42 v42.0.0
 	github.com/olekukonko/tablewriter v0.0.5
+	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
 )
 
 require (
+	github.com/golang/protobuf v1.3.2 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
 )

go.sum

@@ -1,6 +1,7 @@
 github.com/bradleyfalzon/ghinstallation/v2 v2.0.3/go.mod h1:tlgi+JWCXnKFx/Y4WtnDbZEINo31N5bcvnCoqieefmk=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
@@ -18,7 +19,9 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -29,4 +32,5 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=