loading up the forgejo repo on tangled to test page performance
Merge pull request '[Port] container.FilterSlice function (gitea#30339 & gitea#30370)' (#3264) from oliverpool/forgejo:port_30339 into forgejo
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/3264
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
+157
-200
+3
-8
models/actions/run_job_list.go
+3
-8
models/actions/run_job_list.go
···
16
16
type ActionJobList []*ActionRunJob
17
17
18
18
func (jobs ActionJobList) GetRunIDs() []int64 {
19
-
ids := make(container.Set[int64], len(jobs))
20
-
for _, j := range jobs {
21
-
if j.RunID == 0 {
22
-
continue
23
-
}
24
-
ids.Add(j.RunID)
25
-
}
26
-
return ids.Values()
19
+
return container.FilterSlice(jobs, func(j *ActionRunJob) (int64, bool) {
20
+
return j.RunID, j.RunID != 0
21
+
})
27
22
}
28
23
29
24
func (jobs ActionJobList) LoadRuns(ctx context.Context, withRepo bool) error {
+6
-10
models/actions/run_list.go
+6
-10
models/actions/run_list.go
···
19
19
20
20
// GetUserIDs returns a slice of user's id
21
21
func (runs RunList) GetUserIDs() []int64 {
22
-
ids := make(container.Set[int64], len(runs))
23
-
for _, run := range runs {
24
-
ids.Add(run.TriggerUserID)
25
-
}
26
-
return ids.Values()
22
+
return container.FilterSlice(runs, func(run *ActionRun) (int64, bool) {
23
+
return run.TriggerUserID, true
24
+
})
27
25
}
28
26
29
27
func (runs RunList) GetRepoIDs() []int64 {
30
-
ids := make(container.Set[int64], len(runs))
31
-
for _, run := range runs {
32
-
ids.Add(run.RepoID)
33
-
}
34
-
return ids.Values()
28
+
return container.FilterSlice(runs, func(run *ActionRun) (int64, bool) {
29
+
return run.RepoID, true
30
+
})
35
31
}
36
32
37
33
func (runs RunList) LoadTriggerUser(ctx context.Context) error {
+6
-18
models/actions/runner_list.go
+6
-18
models/actions/runner_list.go
···
16
16
17
17
// GetUserIDs returns a slice of user's id
18
18
func (runners RunnerList) GetUserIDs() []int64 {
19
-
ids := make(container.Set[int64], len(runners))
20
-
for _, runner := range runners {
21
-
if runner.OwnerID == 0 {
22
-
continue
23
-
}
24
-
ids.Add(runner.OwnerID)
25
-
}
26
-
return ids.Values()
19
+
return container.FilterSlice(runners, func(runner *ActionRunner) (int64, bool) {
20
+
return runner.OwnerID, runner.OwnerID != 0
21
+
})
27
22
}
28
23
29
24
func (runners RunnerList) LoadOwners(ctx context.Context) error {
···
41
36
}
42
37
43
38
func (runners RunnerList) getRepoIDs() []int64 {
44
-
repoIDs := make(container.Set[int64], len(runners))
45
-
for _, runner := range runners {
46
-
if runner.RepoID == 0 {
47
-
continue
48
-
}
49
-
if _, ok := repoIDs[runner.RepoID]; !ok {
50
-
repoIDs[runner.RepoID] = struct{}{}
51
-
}
52
-
}
53
-
return repoIDs.Values()
39
+
return container.FilterSlice(runners, func(runner *ActionRunner) (int64, bool) {
40
+
return runner.RepoID, runner.RepoID > 0
41
+
})
54
42
}
55
43
56
44
func (runners RunnerList) LoadRepos(ctx context.Context) error {
+6
-10
models/actions/schedule_list.go
+6
-10
models/actions/schedule_list.go
···
18
18
19
19
// GetUserIDs returns a slice of user's id
20
20
func (schedules ScheduleList) GetUserIDs() []int64 {
21
-
ids := make(container.Set[int64], len(schedules))
22
-
for _, schedule := range schedules {
23
-
ids.Add(schedule.TriggerUserID)
24
-
}
25
-
return ids.Values()
21
+
return container.FilterSlice(schedules, func(schedule *ActionSchedule) (int64, bool) {
22
+
return schedule.TriggerUserID, true
23
+
})
26
24
}
27
25
28
26
func (schedules ScheduleList) GetRepoIDs() []int64 {
29
-
ids := make(container.Set[int64], len(schedules))
30
-
for _, schedule := range schedules {
31
-
ids.Add(schedule.RepoID)
32
-
}
33
-
return ids.Values()
27
+
return container.FilterSlice(schedules, func(schedule *ActionSchedule) (int64, bool) {
28
+
return schedule.RepoID, true
29
+
})
34
30
}
35
31
36
32
func (schedules ScheduleList) LoadTriggerUser(ctx context.Context) error {
+6
-10
models/actions/schedule_spec_list.go
+6
-10
models/actions/schedule_spec_list.go
···
16
16
type SpecList []*ActionScheduleSpec
17
17
18
18
func (specs SpecList) GetScheduleIDs() []int64 {
19
-
ids := make(container.Set[int64], len(specs))
20
-
for _, spec := range specs {
21
-
ids.Add(spec.ScheduleID)
22
-
}
23
-
return ids.Values()
19
+
return container.FilterSlice(specs, func(spec *ActionScheduleSpec) (int64, bool) {
20
+
return spec.ScheduleID, true
21
+
})
24
22
}
25
23
26
24
func (specs SpecList) LoadSchedules(ctx context.Context) error {
···
46
44
}
47
45
48
46
func (specs SpecList) GetRepoIDs() []int64 {
49
-
ids := make(container.Set[int64], len(specs))
50
-
for _, spec := range specs {
51
-
ids.Add(spec.RepoID)
52
-
}
53
-
return ids.Values()
47
+
return container.FilterSlice(specs, func(spec *ActionScheduleSpec) (int64, bool) {
48
+
return spec.RepoID, true
49
+
})
54
50
}
55
51
56
52
func (specs SpecList) LoadRepos(ctx context.Context) error {
+3
-8
models/actions/task_list.go
+3
-8
models/actions/task_list.go
···
16
16
type TaskList []*ActionTask
17
17
18
18
func (tasks TaskList) GetJobIDs() []int64 {
19
-
ids := make(container.Set[int64], len(tasks))
20
-
for _, t := range tasks {
21
-
if t.JobID == 0 {
22
-
continue
23
-
}
24
-
ids.Add(t.JobID)
25
-
}
26
-
return ids.Values()
19
+
return container.FilterSlice(tasks, func(t *ActionTask) (int64, bool) {
20
+
return t.JobID, t.JobID != 0
21
+
})
27
22
}
28
23
29
24
func (tasks TaskList) LoadJobs(ctx context.Context) error {
+12
-18
models/activities/action_list.go
+12
-18
models/activities/action_list.go
···
22
22
type ActionList []*Action
23
23
24
24
func (actions ActionList) getUserIDs() []int64 {
25
-
userIDs := make(container.Set[int64], len(actions))
26
-
for _, action := range actions {
27
-
userIDs.Add(action.ActUserID)
28
-
}
29
-
return userIDs.Values()
25
+
return container.FilterSlice(actions, func(action *Action) (int64, bool) {
26
+
return action.ActUserID, true
27
+
})
30
28
}
31
29
32
30
func (actions ActionList) LoadActUsers(ctx context.Context) (map[int64]*user_model.User, error) {
···
50
48
}
51
49
52
50
func (actions ActionList) getRepoIDs() []int64 {
53
-
repoIDs := make(container.Set[int64], len(actions))
54
-
for _, action := range actions {
55
-
repoIDs.Add(action.RepoID)
56
-
}
57
-
return repoIDs.Values()
51
+
return container.FilterSlice(actions, func(action *Action) (int64, bool) {
52
+
return action.RepoID, true
53
+
})
58
54
}
59
55
60
56
func (actions ActionList) LoadRepositories(ctx context.Context) error {
···
80
76
userMap = make(map[int64]*user_model.User)
81
77
}
82
78
83
-
userSet := make(container.Set[int64], len(actions))
84
-
for _, action := range actions {
79
+
missingUserIDs := container.FilterSlice(actions, func(action *Action) (int64, bool) {
85
80
if action.Repo == nil {
86
-
continue
87
-
}
88
-
if _, ok := userMap[action.Repo.OwnerID]; !ok {
89
-
userSet.Add(action.Repo.OwnerID)
81
+
return 0, false
90
82
}
91
-
}
83
+
_, alreadyLoaded := userMap[action.Repo.OwnerID]
84
+
return action.Repo.OwnerID, !alreadyLoaded
85
+
})
92
86
93
87
if err := db.GetEngine(ctx).
94
-
In("id", userSet.Values()).
88
+
In("id", missingUserIDs).
95
89
Find(&userMap); err != nil {
96
90
return fmt.Errorf("find user: %w", err)
97
91
}
+4
-8
models/activities/notification_list.go
+4
-8
models/activities/notification_list.go
···
196
196
return nil
197
197
}
198
198
199
+
// getPendingRepoIDs returns all the repositoty ids which haven't been loaded
199
200
func (nl NotificationList) getPendingRepoIDs() []int64 {
200
-
ids := make(container.Set[int64], len(nl))
201
-
for _, notification := range nl {
202
-
if notification.Repository != nil {
203
-
continue
204
-
}
205
-
ids.Add(notification.RepoID)
206
-
}
207
-
return ids.Values()
201
+
return container.FilterSlice(nl, func(n *Notification) (int64, bool) {
202
+
return n.RepoID, n.Repository == nil
203
+
})
208
204
}
209
205
210
206
// LoadRepos loads repositories from database
+11
-15
models/git/branch_list.go
+11
-15
models/git/branch_list.go
···
17
17
type BranchList []*Branch
18
18
19
19
func (branches BranchList) LoadDeletedBy(ctx context.Context) error {
20
-
ids := container.Set[int64]{}
21
-
for _, branch := range branches {
22
-
if !branch.IsDeleted {
23
-
continue
24
-
}
25
-
ids.Add(branch.DeletedByID)
26
-
}
20
+
ids := container.FilterSlice(branches, func(branch *Branch) (int64, bool) {
21
+
return branch.DeletedByID, branch.IsDeleted
22
+
})
23
+
27
24
usersMap := make(map[int64]*user_model.User, len(ids))
28
-
if err := db.GetEngine(ctx).In("id", ids.Values()).Find(&usersMap); err != nil {
25
+
if err := db.GetEngine(ctx).In("id", ids).Find(&usersMap); err != nil {
29
26
return err
30
27
}
31
28
for _, branch := range branches {
···
41
38
}
42
39
43
40
func (branches BranchList) LoadPusher(ctx context.Context) error {
44
-
ids := container.Set[int64]{}
45
-
for _, branch := range branches {
46
-
if branch.PusherID > 0 { // pusher_id maybe zero because some branches are sync by backend with no pusher
47
-
ids.Add(branch.PusherID)
48
-
}
49
-
}
41
+
ids := container.FilterSlice(branches, func(branch *Branch) (int64, bool) {
42
+
// pusher_id maybe zero because some branches are sync by backend with no pusher
43
+
return branch.PusherID, branch.PusherID > 0
44
+
})
45
+
50
46
usersMap := make(map[int64]*user_model.User, len(ids))
51
-
if err := db.GetEngine(ctx).In("id", ids.Values()).Find(&usersMap); err != nil {
47
+
if err := db.GetEngine(ctx).In("id", ids).Find(&usersMap); err != nil {
52
48
return err
53
49
}
54
50
for _, branch := range branches {
+4
-5
models/issues/comment.go
+4
-5
models/issues/comment.go
···
1289
1289
return nil
1290
1290
}
1291
1291
1292
-
issueIDs := make(container.Set[int64])
1293
-
for _, comment := range comments {
1294
-
issueIDs.Add(comment.IssueID)
1295
-
}
1292
+
issueIDs := container.FilterSlice(comments, func(comment *Comment) (int64, bool) {
1293
+
return comment.IssueID, true
1294
+
})
1296
1295
1297
1296
ctx, committer, err := db.TxContext(ctx)
1298
1297
if err != nil {
···
1315
1314
}
1316
1315
}
1317
1316
1318
-
for issueID := range issueIDs {
1317
+
for _, issueID := range issueIDs {
1319
1318
if _, err := db.Exec(ctx, "UPDATE issue set num_comments = (SELECT count(*) FROM comment WHERE issue_id = ? AND `type`=?) WHERE id = ?",
1320
1319
issueID, CommentTypeComment, issueID); err != nil {
1321
1320
return err
+28
-65
models/issues/comment_list.go
+28
-65
models/issues/comment_list.go
···
17
17
type CommentList []*Comment
18
18
19
19
func (comments CommentList) getPosterIDs() []int64 {
20
-
posterIDs := make(container.Set[int64], len(comments))
21
-
for _, comment := range comments {
22
-
if comment.PosterID > 0 {
23
-
posterIDs.Add(comment.PosterID)
24
-
}
25
-
}
26
-
return posterIDs.Values()
20
+
return container.FilterSlice(comments, func(c *Comment) (int64, bool) {
21
+
return c.PosterID, c.PosterID > 0
22
+
})
27
23
}
28
24
29
25
// LoadPosters loads posters
···
44
40
}
45
41
46
42
func (comments CommentList) getLabelIDs() []int64 {
47
-
ids := make(container.Set[int64], len(comments))
48
-
for _, comment := range comments {
49
-
if comment.LabelID > 0 {
50
-
ids.Add(comment.LabelID)
51
-
}
52
-
}
53
-
return ids.Values()
43
+
return container.FilterSlice(comments, func(comment *Comment) (int64, bool) {
44
+
return comment.LabelID, comment.LabelID > 0
45
+
})
54
46
}
55
47
56
48
func (comments CommentList) loadLabels(ctx context.Context) error {
···
94
86
}
95
87
96
88
func (comments CommentList) getMilestoneIDs() []int64 {
97
-
ids := make(container.Set[int64], len(comments))
98
-
for _, comment := range comments {
99
-
if comment.MilestoneID > 0 {
100
-
ids.Add(comment.MilestoneID)
101
-
}
102
-
}
103
-
return ids.Values()
89
+
return container.FilterSlice(comments, func(comment *Comment) (int64, bool) {
90
+
return comment.MilestoneID, comment.MilestoneID > 0
91
+
})
104
92
}
105
93
106
94
func (comments CommentList) loadMilestones(ctx context.Context) error {
···
137
125
}
138
126
139
127
func (comments CommentList) getOldMilestoneIDs() []int64 {
140
-
ids := make(container.Set[int64], len(comments))
141
-
for _, comment := range comments {
142
-
if comment.OldMilestoneID > 0 {
143
-
ids.Add(comment.OldMilestoneID)
144
-
}
145
-
}
146
-
return ids.Values()
128
+
return container.FilterSlice(comments, func(comment *Comment) (int64, bool) {
129
+
return comment.OldMilestoneID, comment.OldMilestoneID > 0
130
+
})
147
131
}
148
132
149
133
func (comments CommentList) loadOldMilestones(ctx context.Context) error {
···
180
164
}
181
165
182
166
func (comments CommentList) getAssigneeIDs() []int64 {
183
-
ids := make(container.Set[int64], len(comments))
184
-
for _, comment := range comments {
185
-
if comment.AssigneeID > 0 {
186
-
ids.Add(comment.AssigneeID)
187
-
}
188
-
}
189
-
return ids.Values()
167
+
return container.FilterSlice(comments, func(comment *Comment) (int64, bool) {
168
+
return comment.AssigneeID, comment.AssigneeID > 0
169
+
})
190
170
}
191
171
192
172
func (comments CommentList) loadAssignees(ctx context.Context) error {
···
237
217
238
218
// getIssueIDs returns all the issue ids on this comment list which issue hasn't been loaded
239
219
func (comments CommentList) getIssueIDs() []int64 {
240
-
ids := make(container.Set[int64], len(comments))
241
-
for _, comment := range comments {
242
-
if comment.Issue != nil {
243
-
continue
244
-
}
245
-
ids.Add(comment.IssueID)
246
-
}
247
-
return ids.Values()
220
+
return container.FilterSlice(comments, func(comment *Comment) (int64, bool) {
221
+
return comment.IssueID, comment.Issue == nil
222
+
})
248
223
}
249
224
250
225
// Issues returns all the issues of comments
···
311
286
}
312
287
313
288
func (comments CommentList) getDependentIssueIDs() []int64 {
314
-
ids := make(container.Set[int64], len(comments))
315
-
for _, comment := range comments {
289
+
return container.FilterSlice(comments, func(comment *Comment) (int64, bool) {
316
290
if comment.DependentIssue != nil {
317
-
continue
291
+
return 0, false
318
292
}
319
-
if comment.DependentIssueID > 0 {
320
-
ids.Add(comment.DependentIssueID)
321
-
}
322
-
}
323
-
return ids.Values()
293
+
return comment.DependentIssueID, comment.DependentIssueID > 0
294
+
})
324
295
}
325
296
326
297
func (comments CommentList) loadDependentIssues(ctx context.Context) error {
···
375
346
376
347
// getAttachmentCommentIDs only return the comment ids which possibly has attachments
377
348
func (comments CommentList) getAttachmentCommentIDs() []int64 {
378
-
ids := make(container.Set[int64], len(comments))
379
-
for _, comment := range comments {
380
-
if comment.Type.HasAttachmentSupport() {
381
-
ids.Add(comment.ID)
382
-
}
383
-
}
384
-
return ids.Values()
349
+
return container.FilterSlice(comments, func(comment *Comment) (int64, bool) {
350
+
return comment.ID, comment.Type.HasAttachmentSupport()
351
+
})
385
352
}
386
353
387
354
// LoadAttachmentsByIssue loads attachments by issue id
···
449
416
}
450
417
451
418
func (comments CommentList) getReviewIDs() []int64 {
452
-
ids := make(container.Set[int64], len(comments))
453
-
for _, comment := range comments {
454
-
if comment.ReviewID > 0 {
455
-
ids.Add(comment.ReviewID)
456
-
}
457
-
}
458
-
return ids.Values()
419
+
return container.FilterSlice(comments, func(comment *Comment) (int64, bool) {
420
+
return comment.ReviewID, comment.ReviewID > 0
421
+
})
459
422
}
460
423
461
424
func (comments CommentList) loadReviews(ctx context.Context) error {
+6
-10
models/issues/issue_list.go
+6
-10
models/issues/issue_list.go
···
74
74
}
75
75
76
76
func (issues IssueList) getPosterIDs() []int64 {
77
-
posterIDs := make(container.Set[int64], len(issues))
78
-
for _, issue := range issues {
79
-
posterIDs.Add(issue.PosterID)
80
-
}
81
-
return posterIDs.Values()
77
+
return container.FilterSlice(issues, func(issue *Issue) (int64, bool) {
78
+
return issue.PosterID, true
79
+
})
82
80
}
83
81
84
82
func (issues IssueList) loadPosters(ctx context.Context) error {
···
193
191
}
194
192
195
193
func (issues IssueList) getMilestoneIDs() []int64 {
196
-
ids := make(container.Set[int64], len(issues))
197
-
for _, issue := range issues {
198
-
ids.Add(issue.MilestoneID)
199
-
}
200
-
return ids.Values()
194
+
return container.FilterSlice(issues, func(issue *Issue) (int64, bool) {
195
+
return issue.MilestoneID, true
196
+
})
201
197
}
202
198
203
199
func (issues IssueList) loadMilestones(ctx context.Context) error {
+4
-6
models/issues/reaction.go
+4
-6
models/issues/reaction.go
···
305
305
}
306
306
307
307
func (list ReactionList) getUserIDs() []int64 {
308
-
userIDs := make(container.Set[int64], len(list))
309
-
for _, reaction := range list {
308
+
return container.FilterSlice(list, func(reaction *Reaction) (int64, bool) {
310
309
if reaction.OriginalAuthor != "" {
311
-
continue
310
+
return 0, false
312
311
}
313
-
userIDs.Add(reaction.UserID)
314
-
}
315
-
return userIDs.Values()
312
+
return reaction.UserID, true
313
+
})
316
314
}
317
315
318
316
func valuesUser(m map[int64]*user_model.User) []*user_model.User {
+4
-5
models/issues/review_list.go
+4
-5
models/issues/review_list.go
···
38
38
}
39
39
40
40
func (reviews ReviewList) LoadIssues(ctx context.Context) error {
41
-
issueIDs := container.Set[int64]{}
42
-
for i := 0; i < len(reviews); i++ {
43
-
issueIDs.Add(reviews[i].IssueID)
44
-
}
41
+
issueIDs := container.FilterSlice(reviews, func(review *Review) (int64, bool) {
42
+
return review.IssueID, true
43
+
})
45
44
46
-
issues, err := GetIssuesByIDs(ctx, issueIDs.Values())
45
+
issues, err := GetIssuesByIDs(ctx, issueIDs)
47
46
if err != nil {
48
47
return err
49
48
}
+5
-4
models/repo/repo_list.go
+5
-4
models/repo/repo_list.go
···
104
104
return nil
105
105
}
106
106
107
-
set := make(container.Set[int64])
107
+
userIDs := container.FilterSlice(repos, func(repo *Repository) (int64, bool) {
108
+
return repo.OwnerID, true
109
+
})
108
110
repoIDs := make([]int64, len(repos))
109
111
for i := range repos {
110
-
set.Add(repos[i].OwnerID)
111
112
repoIDs[i] = repos[i].ID
112
113
}
113
114
114
115
// Load owners.
115
-
users := make(map[int64]*user_model.User, len(set))
116
+
users := make(map[int64]*user_model.User, len(userIDs))
116
117
if err := db.GetEngine(ctx).
117
118
Where("id > 0").
118
-
In("id", set.Values()).
119
+
In("id", userIDs).
119
120
Find(&users); err != nil {
120
121
return fmt.Errorf("find users: %w", err)
121
122
}
+21
modules/container/filter.go
+21
modules/container/filter.go
···
1
+
// Copyright 2024 The Gitea Authors. All rights reserved.
2
+
// SPDX-License-Identifier: MIT
3
+
4
+
package container
5
+
6
+
import "slices"
7
+
8
+
// FilterSlice ranges over the slice and calls include() for each element.
9
+
// If the second returned value is true, the first returned value will be included in the resulting
10
+
// slice (after deduplication).
11
+
func FilterSlice[E any, T comparable](s []E, include func(E) (T, bool)) []T {
12
+
filtered := make([]T, 0, len(s)) // slice will be clipped before returning
13
+
seen := make(map[T]bool, len(s))
14
+
for i := range s {
15
+
if v, ok := include(s[i]); ok && !seen[v] {
16
+
filtered = append(filtered, v)
17
+
seen[v] = true
18
+
}
19
+
}
20
+
return slices.Clip(filtered)
21
+
}
+28
modules/container/filter_test.go
+28
modules/container/filter_test.go
···
1
+
// Copyright 2024 The Gitea Authors. All rights reserved.
2
+
// SPDX-License-Identifier: MIT
3
+
4
+
package container
5
+
6
+
import (
7
+
"testing"
8
+
9
+
"github.com/stretchr/testify/assert"
10
+
)
11
+
12
+
func TestFilterMapUnique(t *testing.T) {
13
+
result := FilterSlice([]int{
14
+
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
15
+
}, func(i int) (int, bool) {
16
+
switch i {
17
+
case 0:
18
+
return 0, true // included later
19
+
case 1:
20
+
return 0, true // duplicate of previous (should be ignored)
21
+
case 2:
22
+
return 2, false // not included
23
+
default:
24
+
return i, true
25
+
}
26
+
})
27
+
assert.Equal(t, []int{0, 3, 4, 5, 6, 7, 8, 9}, result)
28
+
}