Skip to content

Commit bacb88c

Browse files
committed
Explicitly meter calls to Github API. Remove retrier.
We do this because the Github API triggers abuse warnings if a single user makes lots of concurrent requests. By having a shared ticker, we ensure that Github API requests don't happen in parallel. Moreover, we also ensure that there's a global rate limit of 1 request to the GithubAPI per 720 milliseconds. We're following the approach shown here: google/go-github#431 This means that we'll stay within the rate limit requirement of 5000 requests-per-hour for authenticated users. ``` 5000 QPH = 83.3 QPM = 1.38 QPS = 720ms/query ```
1 parent f022324 commit bacb88c

5 files changed

Lines changed: 36 additions & 75 deletions

File tree

‎cmd/merge.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ func mergeOneRepo(r initialize.Repo, ctx context.Context) error {
6060
PRNumber: prNumber,
6161
CommitSHA: pushOutput.CommitSHA,
6262
}
63-
output, err := merge.Merge(ctx, input)
63+
output, err := merge.Merge(ctx, input, githubLimiter)
6464
if err != nil {
6565
o := struct {
6666
merge.Output

‎cmd/push.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ func pushOneRepo(r initialize.Repo, ctx context.Context) error {
7575
BranchName: planOutput.BranchName,
7676
RepoOwner: r.Owner,
7777
}
78-
output, err := push.Push(ctx, input)
78+
output, err := push.Push(ctx, input, githubLimiter)
7979
if err != nil {
8080
o := struct {
8181
push.Output

‎cmd/root.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"os"
77
"path"
88
"path/filepath"
9+
"time"
910

1011
"github.com/Clever/microplane/initialize"
1112
"github.com/spf13/cobra"
@@ -14,6 +15,10 @@ import (
1415
var workDir string
1516
var cliVersion string
1617

18+
// Github's rate limit for authenticated requests is 5000 QPH = 83.3 QPM = 1.38 QPS = 720ms/query
19+
// We also use a global limiter to prevent concurrent requests, which trigger Github's abuse detection
20+
var githubLimiter = time.NewTicker(720 * time.Millisecond)
21+
1722
var rootCmd = &cobra.Command{
1823
Use: "mp",
1924
Short: "Microplane makes git changes across many repos",

‎merge/merge.go‎

Lines changed: 11 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,10 @@ import (
66
"os"
77
"time"
88

9-
"github.com/eapache/go-resiliency/retrier"
109
"github.com/google/go-github/github"
1110
"golang.org/x/oauth2"
1211
)
1312

14-
// Command represents a command to run.
15-
type Command struct {
16-
Path string
17-
Args []string
18-
}
19-
2013
// Input to Push()
2114
type Input struct {
2215
// Org on Github, e.g. "Clever"
@@ -42,30 +35,19 @@ type Error struct {
4235
}
4336

4437
// Merge an open PR in Github
45-
func Merge(ctx context.Context, input Input) (Output, error) {
38+
func Merge(ctx context.Context, input Input, githubLimiter *time.Ticker) (Output, error) {
39+
// Create Github Client
4640
ts := oauth2.StaticTokenSource(
4741
&oauth2.Token{AccessToken: os.Getenv("GITHUB_API_TOKEN")},
4842
)
4943
tc := oauth2.NewClient(ctx, ts)
50-
5144
client := github.NewClient(tc)
5245

53-
// Configure a retrier - exponential backoff upon hitting Github API rate limit errors
54-
r := retrier.New(retrier.ExponentialBackoff(10, time.Second), retrier.WhitelistClassifier{
55-
&github.RateLimitError{},
56-
&github.AbuseRateLimitError{},
57-
})
58-
r.SetJitter(0.5)
59-
6046
// OK to merge?
6147

6248
// (1) Check if the PR is mergeable
63-
var pr *github.PullRequest
64-
err := r.Run(func() error {
65-
var prErr error
66-
pr, _, prErr = client.PullRequests.Get(ctx, input.Org, input.Repo, input.PRNumber)
67-
return prErr
68-
})
49+
<-githubLimiter.C
50+
pr, _, err := client.PullRequests.Get(ctx, input.Org, input.Repo, input.PRNumber)
6951
if err != nil {
7052
return Output{Success: false}, err
7153
}
@@ -75,13 +57,9 @@ func Merge(ctx context.Context, input Input) (Output, error) {
7557
}
7658

7759
// (2) Check commit status
78-
var status *github.CombinedStatus
79-
err = r.Run(func() error {
80-
var csErr error
81-
opt := &github.ListOptions{}
82-
status, _, csErr = client.Repositories.GetCombinedStatus(ctx, input.Org, input.Repo, input.CommitSHA, opt)
83-
return csErr
84-
})
60+
opt := &github.ListOptions{}
61+
<-githubLimiter.C
62+
status, _, err := client.Repositories.GetCombinedStatus(ctx, input.Org, input.Repo, input.CommitSHA, opt)
8563
if err != nil {
8664
return Output{Success: false}, err
8765
}
@@ -97,12 +75,8 @@ func Merge(ctx context.Context, input Input) (Output, error) {
9775
// Merge the PR
9876
options := &github.PullRequestOptions{}
9977
commitMsg := ""
100-
var result *github.PullRequestMergeResult
101-
err = r.Run(func() error {
102-
var mergeErr error
103-
result, _, mergeErr = client.PullRequests.Merge(ctx, input.Org, input.Repo, input.PRNumber, commitMsg, options)
104-
return mergeErr
105-
})
78+
<-githubLimiter.C
79+
result, _, err := client.PullRequests.Merge(ctx, input.Org, input.Repo, input.PRNumber, commitMsg, options)
10680
if err != nil {
10781
return Output{Success: false}, err
10882
}
@@ -112,10 +86,8 @@ func Merge(ctx context.Context, input Input) (Output, error) {
11286
}
11387

11488
// Delete the branch
115-
err = r.Run(func() error {
116-
_, deleteErr := client.Git.DeleteRef(ctx, input.Org, input.Repo, "heads/"+*pr.Head.Ref)
117-
return deleteErr
118-
})
89+
<-githubLimiter.C
90+
_, err = client.Git.DeleteRef(ctx, input.Org, input.Repo, "heads/"+*pr.Head.Ref)
11991
if err != nil {
12092
return Output{Success: false}, err
12193
}

‎push/push.go‎

Lines changed: 18 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ import (
1010
"strings"
1111
"time"
1212

13-
"github.com/eapache/go-resiliency/retrier"
14-
"github.com/google/go-github/github"
1513
"golang.org/x/oauth2"
14+
15+
"github.com/google/go-github/github"
1616
)
1717

1818
// Command represents a command to run.
@@ -71,7 +71,7 @@ func (o Output) String() string {
7171
}
7272

7373
// Push pushes the commit to Github and opens a pull request
74-
func Push(ctx context.Context, input Input) (Output, error) {
74+
func Push(ctx context.Context, input Input, githubLimiter *time.Ticker) (Output, error) {
7575
// Get the commit SHA from the last commit
7676
cmd := Command{Path: "git", Args: []string{"log", "-1", "--pretty=format:%H"}}
7777
gitLog := exec.CommandContext(ctx, cmd.Path, cmd.Args...)
@@ -90,54 +90,36 @@ func Push(ctx context.Context, input Input) (Output, error) {
9090
return Output{Success: false}, errors.New(string(output))
9191
}
9292

93-
// Open a pull request, if one doesn't exist already
93+
// Create Github Client
9494
ts := oauth2.StaticTokenSource(
9595
&oauth2.Token{AccessToken: os.Getenv("GITHUB_API_TOKEN")},
9696
)
9797
tc := oauth2.NewClient(ctx, ts)
9898
client := github.NewClient(tc)
99+
100+
// Open a pull request, if one doesn't exist already
99101
head := fmt.Sprintf("%s:%s", input.RepoOwner, input.BranchName)
100102
base := "master"
101103

102-
// Configure a retrier - exponential backoff upon hitting Github API rate limit errors
103-
r := retrier.New(retrier.ExponentialBackoff(10, time.Second), retrier.WhitelistClassifier{
104-
&github.RateLimitError{},
105-
&github.AbuseRateLimitError{},
106-
})
107-
r.SetJitter(0.5)
108-
109-
var pr *github.PullRequest
110-
err = r.Run(func() error {
111-
var prErr error
112-
pr, prErr = findOrCreatePR(ctx, client, input.RepoOwner, input.RepoName, &github.NewPullRequest{
113-
Title: &input.PRMessage,
114-
Head: &head,
115-
Base: &base,
116-
})
117-
return prErr
118-
})
104+
pr, err := findOrCreatePR(ctx, client, input.RepoOwner, input.RepoName, &github.NewPullRequest{
105+
Title: &input.PRMessage,
106+
Head: &head,
107+
Base: &base,
108+
}, githubLimiter)
119109
if err != nil {
120110
return Output{Success: false}, err
121111
}
122112

123113
if pr.Assignee == nil || pr.Assignee.Login == nil || *pr.Assignee.Login != input.PRAssignee {
124-
err = r.Run(func() error {
125-
var addErr error
126-
_, _, addErr = client.Issues.AddAssignees(ctx, input.RepoOwner, input.RepoName,
127-
*pr.Number, []string{input.PRAssignee})
128-
return addErr
129-
})
114+
<-githubLimiter.C
115+
_, _, err := client.Issues.AddAssignees(ctx, input.RepoOwner, input.RepoName, *pr.Number, []string{input.PRAssignee})
130116
if err != nil {
131117
return Output{Success: false}, err
132118
}
133119
}
134120

135-
var cs *github.CombinedStatus
136-
err = r.Run(func() error {
137-
var getErr error
138-
cs, _, getErr = client.Repositories.GetCombinedStatus(ctx, input.RepoOwner, input.RepoName, *pr.Head.SHA, nil)
139-
return getErr
140-
})
121+
<-githubLimiter.C
122+
cs, _, err := client.Repositories.GetCombinedStatus(ctx, input.RepoOwner, input.RepoName, *pr.Head.SHA, nil)
141123
if err != nil {
142124
return Output{Success: false}, err
143125
}
@@ -171,10 +153,12 @@ func Push(ctx context.Context, input Input) (Output, error) {
171153
}, nil
172154
}
173155

174-
func findOrCreatePR(ctx context.Context, client *github.Client, owner string, name string, pull *github.NewPullRequest) (*github.PullRequest, error) {
156+
func findOrCreatePR(ctx context.Context, client *github.Client, owner string, name string, pull *github.NewPullRequest, githubLimiter *time.Ticker) (*github.PullRequest, error) {
175157
var pr *github.PullRequest
158+
<-githubLimiter.C
176159
newPR, _, err := client.PullRequests.Create(ctx, owner, name, pull)
177160
if err != nil && strings.Contains(err.Error(), "pull request already exists") {
161+
<-githubLimiter.C
178162
existingPRs, _, err := client.PullRequests.List(ctx, owner, name, &github.PullRequestListOptions{
179163
Head: *pull.Head,
180164
Base: *pull.Base,

0 commit comments

Comments
 (0)