loading up the forgejo repo on tangled to test page performance
Improve retrying index issues (#27554)
Fix #27540
authored by
Jason Song
and committed by
GitHub
1be49fdd
cddf245c
+52
-40
+7
-18
modules/indexer/issues/indexer.go
+7
-18
modules/indexer/issues/indexer.go
···
204
204
func populateIssueIndexer(ctx context.Context) {
205
205
ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Service: PopulateIssueIndexer", process.SystemProcessType, true)
206
206
defer finished()
207
-
if err := PopulateIssueIndexer(ctx, true); err != nil {
207
+
ctx = contextWithKeepRetry(ctx) // keep retrying since it's a background task
208
+
if err := PopulateIssueIndexer(ctx); err != nil {
208
209
log.Error("Issue indexer population failed: %v", err)
209
210
}
210
211
}
211
212
212
-
func PopulateIssueIndexer(ctx context.Context, keepRetrying bool) error {
213
+
func PopulateIssueIndexer(ctx context.Context) error {
213
214
for page := 1; ; page++ {
214
215
select {
215
216
case <-ctx.Done():
···
232
233
}
233
234
234
235
for _, repo := range repos {
235
-
for {
236
-
select {
237
-
case <-ctx.Done():
238
-
return fmt.Errorf("shutdown before completion: %w", ctx.Err())
239
-
default:
240
-
}
241
-
if err := updateRepoIndexer(ctx, repo.ID); err != nil {
242
-
if keepRetrying && ctx.Err() == nil {
243
-
log.Warn("Retry to populate issue indexer for repo %d: %v", repo.ID, err)
244
-
continue
245
-
}
246
-
return fmt.Errorf("populate issue indexer for repo %d: %v", repo.ID, err)
247
-
}
248
-
break
236
+
if err := updateRepoIndexer(ctx, repo.ID); err != nil {
237
+
return fmt.Errorf("populate issue indexer for repo %d: %v", repo.ID, err)
249
238
}
250
239
}
251
240
}
···
259
248
}
260
249
261
250
// UpdateIssueIndexer add/update an issue to the issue indexer
262
-
func UpdateIssueIndexer(issueID int64) {
263
-
if err := updateIssueIndexer(issueID); err != nil {
251
+
func UpdateIssueIndexer(ctx context.Context, issueID int64) {
252
+
if err := updateIssueIndexer(ctx, issueID); err != nil {
264
253
log.Error("Unable to push issue %d to issue indexer: %v", issueID, err)
265
254
}
266
255
}
+34
-12
modules/indexer/issues/util.go
+34
-12
modules/indexer/issues/util.go
···
127
127
return fmt.Errorf("issue_model.GetIssueIDsByRepoID: %w", err)
128
128
}
129
129
for _, id := range ids {
130
-
if err := updateIssueIndexer(id); err != nil {
130
+
if err := updateIssueIndexer(ctx, id); err != nil {
131
131
return err
132
132
}
133
133
}
134
134
return nil
135
135
}
136
136
137
-
func updateIssueIndexer(issueID int64) error {
138
-
return pushIssueIndexerQueue(&IndexerMetadata{ID: issueID})
137
+
func updateIssueIndexer(ctx context.Context, issueID int64) error {
138
+
return pushIssueIndexerQueue(ctx, &IndexerMetadata{ID: issueID})
139
139
}
140
140
141
141
func deleteRepoIssueIndexer(ctx context.Context, repoID int64) error {
···
148
148
if len(ids) == 0 {
149
149
return nil
150
150
}
151
-
return pushIssueIndexerQueue(&IndexerMetadata{
151
+
return pushIssueIndexerQueue(ctx, &IndexerMetadata{
152
152
IDs: ids,
153
153
IsDelete: true,
154
154
})
155
155
}
156
156
157
-
func pushIssueIndexerQueue(data *IndexerMetadata) error {
157
+
type keepRetryKey struct{}
158
+
159
+
// contextWithKeepRetry returns a context with a key indicating that the indexer should keep retrying.
160
+
// Please note that it's for background tasks only, and it should not be used for user requests, or it may cause blocking.
161
+
func contextWithKeepRetry(ctx context.Context) context.Context {
162
+
return context.WithValue(ctx, keepRetryKey{}, true)
163
+
}
164
+
165
+
func pushIssueIndexerQueue(ctx context.Context, data *IndexerMetadata) error {
158
166
if issueIndexerQueue == nil {
159
167
// Some unit tests will trigger indexing, but the queue is not initialized.
160
168
// It's OK to ignore it, but log a warning message in case it's not a unit test.
···
162
170
return nil
163
171
}
164
172
165
-
err := issueIndexerQueue.Push(data)
166
-
if errors.Is(err, queue.ErrAlreadyInQueue) {
167
-
return nil
168
-
}
169
-
if errors.Is(err, context.DeadlineExceeded) {
170
-
log.Warn("It seems that issue indexer is slow and the queue is full. Please check the issue indexer or increase the queue size.")
173
+
for {
174
+
select {
175
+
case <-ctx.Done():
176
+
return ctx.Err()
177
+
default:
178
+
}
179
+
err := issueIndexerQueue.Push(data)
180
+
if errors.Is(err, queue.ErrAlreadyInQueue) {
181
+
return nil
182
+
}
183
+
if errors.Is(err, context.DeadlineExceeded) { // the queue is full
184
+
log.Warn("It seems that issue indexer is slow and the queue is full. Please check the issue indexer or increase the queue size.")
185
+
if ctx.Value(keepRetryKey{}) == nil {
186
+
return err
187
+
}
188
+
// It will be better to increase the queue size instead of retrying, but users may ignore the previous warning message.
189
+
// However, even it retries, it may still cause index loss when there's a deadline in the context.
190
+
log.Debug("Retry to push %+v to issue indexer queue", data)
191
+
continue
192
+
}
193
+
return err
171
194
}
172
-
return err
173
195
}
+1
-1
services/cron/tasks_extended.go
+1
-1
services/cron/tasks_extended.go
+8
-8
services/indexer/notify.go
+8
-8
services/indexer/notify.go
···
36
36
func (r *indexerNotifier) CreateIssueComment(ctx context.Context, doer *user_model.User, repo *repo_model.Repository,
37
37
issue *issues_model.Issue, comment *issues_model.Comment, mentions []*user_model.User,
38
38
) {
39
-
issue_indexer.UpdateIssueIndexer(issue.ID)
39
+
issue_indexer.UpdateIssueIndexer(ctx, issue.ID)
40
40
}
41
41
42
42
func (r *indexerNotifier) NewIssue(ctx context.Context, issue *issues_model.Issue, mentions []*user_model.User) {
43
-
issue_indexer.UpdateIssueIndexer(issue.ID)
43
+
issue_indexer.UpdateIssueIndexer(ctx, issue.ID)
44
44
}
45
45
46
46
func (r *indexerNotifier) NewPullRequest(ctx context.Context, pr *issues_model.PullRequest, mentions []*user_model.User) {
···
48
48
log.Error("LoadIssue: %v", err)
49
49
return
50
50
}
51
-
issue_indexer.UpdateIssueIndexer(pr.Issue.ID)
51
+
issue_indexer.UpdateIssueIndexer(ctx, pr.Issue.ID)
52
52
}
53
53
54
54
func (r *indexerNotifier) UpdateComment(ctx context.Context, doer *user_model.User, c *issues_model.Comment, oldContent string) {
···
56
56
log.Error("LoadIssue: %v", err)
57
57
return
58
58
}
59
-
issue_indexer.UpdateIssueIndexer(c.Issue.ID)
59
+
issue_indexer.UpdateIssueIndexer(ctx, c.Issue.ID)
60
60
}
61
61
62
62
func (r *indexerNotifier) DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) {
···
64
64
log.Error("LoadIssue: %v", err)
65
65
return
66
66
}
67
-
issue_indexer.UpdateIssueIndexer(comment.Issue.ID)
67
+
issue_indexer.UpdateIssueIndexer(ctx, comment.Issue.ID)
68
68
}
69
69
70
70
func (r *indexerNotifier) DeleteRepository(ctx context.Context, doer *user_model.User, repo *repo_model.Repository) {
···
120
120
}
121
121
122
122
func (r *indexerNotifier) IssueChangeContent(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, oldContent string) {
123
-
issue_indexer.UpdateIssueIndexer(issue.ID)
123
+
issue_indexer.UpdateIssueIndexer(ctx, issue.ID)
124
124
}
125
125
126
126
func (r *indexerNotifier) IssueChangeTitle(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, oldTitle string) {
127
-
issue_indexer.UpdateIssueIndexer(issue.ID)
127
+
issue_indexer.UpdateIssueIndexer(ctx, issue.ID)
128
128
}
129
129
130
130
func (r *indexerNotifier) IssueChangeRef(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, oldRef string) {
131
-
issue_indexer.UpdateIssueIndexer(issue.ID)
131
+
issue_indexer.UpdateIssueIndexer(ctx, issue.ID)
132
132
}
+2
-1
tests/integration/issue_test.go
+2
-1
tests/integration/issue_test.go
···
4
4
package integration
5
5
6
6
import (
7
+
"context"
7
8
"fmt"
8
9
"net/http"
9
10
"net/url"
···
99
100
RepoID: repo.ID,
100
101
Index: 1,
101
102
})
102
-
issues.UpdateIssueIndexer(issue.ID)
103
+
issues.UpdateIssueIndexer(context.Background(), issue.ID)
103
104
time.Sleep(time.Second * 1)
104
105
const keyword = "first"
105
106
req := NewRequestf(t, "GET", "%s/issues?q=%s", repo.Link(), keyword)