loading up the forgejo repo on tangled to test page performance
Merge pull request 'Add commit status summary table to reduce query from commit status table' (#3245) from viceice/forgejo:feat/commit-status-summary into forgejo
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/3245
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
+225
-30
+9
-12
models/git/commit_status.go
+9
-12
models/git/commit_status.go
···
257
257
}
258
258
259
259
// GetLatestCommitStatusForPairs returns all statuses with a unique context for a given list of repo-sha pairs
260
-
func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHAs map[int64]string, listOptions db.ListOptions) (map[int64][]*CommitStatus, error) {
260
+
func GetLatestCommitStatusForPairs(ctx context.Context, repoSHAs []RepoSHA) (map[int64][]*CommitStatus, error) {
261
261
type result struct {
262
262
Index int64
263
263
RepoID int64
264
+
SHA string
264
265
}
265
266
266
-
results := make([]result, 0, len(repoIDsToLatestCommitSHAs))
267
+
results := make([]result, 0, len(repoSHAs))
267
268
268
269
getBase := func() *xorm.Session {
269
270
return db.GetEngine(ctx).Table(&CommitStatus{})
270
271
}
271
272
272
273
// Create a disjunction of conditions for each repoID and SHA pair
273
-
conds := make([]builder.Cond, 0, len(repoIDsToLatestCommitSHAs))
274
-
for repoID, sha := range repoIDsToLatestCommitSHAs {
275
-
conds = append(conds, builder.Eq{"repo_id": repoID, "sha": sha})
274
+
conds := make([]builder.Cond, 0, len(repoSHAs))
275
+
for _, repoSHA := range repoSHAs {
276
+
conds = append(conds, builder.Eq{"repo_id": repoSHA.RepoID, "sha": repoSHA.SHA})
276
277
}
277
278
sess := getBase().Where(builder.Or(conds...)).
278
-
Select("max( `index` ) as `index`, repo_id").
279
-
GroupBy("context_hash, repo_id").OrderBy("max( `index` ) desc")
280
-
281
-
if !listOptions.IsListAll() {
282
-
sess = db.SetSessionPagination(sess, &listOptions)
283
-
}
279
+
Select("max( `index` ) as `index`, repo_id, sha").
280
+
GroupBy("context_hash, repo_id, sha").OrderBy("max( `index` ) desc")
284
281
285
282
err := sess.Find(&results)
286
283
if err != nil {
···
297
294
cond := builder.Eq{
298
295
"`index`": result.Index,
299
296
"repo_id": result.RepoID,
300
-
"sha": repoIDsToLatestCommitSHAs[result.RepoID],
297
+
"sha": result.SHA,
301
298
}
302
299
conds = append(conds, cond)
303
300
}
+88
models/git/commit_status_summary.go
+88
models/git/commit_status_summary.go
···
1
+
// Copyright 2024 Gitea. All rights reserved.
2
+
// SPDX-License-Identifier: MIT
3
+
4
+
package git
5
+
6
+
import (
7
+
"context"
8
+
9
+
"code.gitea.io/gitea/models/db"
10
+
"code.gitea.io/gitea/modules/setting"
11
+
api "code.gitea.io/gitea/modules/structs"
12
+
13
+
"xorm.io/builder"
14
+
)
15
+
16
+
// CommitStatusSummary holds the latest commit Status of a single Commit
17
+
type CommitStatusSummary struct {
18
+
ID int64 `xorm:"pk autoincr"`
19
+
RepoID int64 `xorm:"INDEX UNIQUE(repo_id_sha)"`
20
+
SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_id_sha)"`
21
+
State api.CommitStatusState `xorm:"VARCHAR(7) NOT NULL"`
22
+
TargetURL string `xorm:"TEXT"`
23
+
}
24
+
25
+
func init() {
26
+
db.RegisterModel(new(CommitStatusSummary))
27
+
}
28
+
29
+
type RepoSHA struct {
30
+
RepoID int64
31
+
SHA string
32
+
}
33
+
34
+
func GetLatestCommitStatusForRepoAndSHAs(ctx context.Context, repoSHAs []RepoSHA) ([]*CommitStatus, error) {
35
+
cond := builder.NewCond()
36
+
for _, rs := range repoSHAs {
37
+
cond = cond.Or(builder.Eq{"repo_id": rs.RepoID, "sha": rs.SHA})
38
+
}
39
+
40
+
var summaries []CommitStatusSummary
41
+
if err := db.GetEngine(ctx).Where(cond).Find(&summaries); err != nil {
42
+
return nil, err
43
+
}
44
+
45
+
commitStatuses := make([]*CommitStatus, 0, len(repoSHAs))
46
+
for _, summary := range summaries {
47
+
commitStatuses = append(commitStatuses, &CommitStatus{
48
+
RepoID: summary.RepoID,
49
+
SHA: summary.SHA,
50
+
State: summary.State,
51
+
TargetURL: summary.TargetURL,
52
+
})
53
+
}
54
+
return commitStatuses, nil
55
+
}
56
+
57
+
func UpdateCommitStatusSummary(ctx context.Context, repoID int64, sha string) error {
58
+
commitStatuses, _, err := GetLatestCommitStatus(ctx, repoID, sha, db.ListOptionsAll)
59
+
if err != nil {
60
+
return err
61
+
}
62
+
state := CalcCommitStatus(commitStatuses)
63
+
// mysql will return 0 when update a record which state hasn't been changed which behaviour is different from other database,
64
+
// so we need to use insert in on duplicate
65
+
if setting.Database.Type.IsMySQL() {
66
+
_, err := db.GetEngine(ctx).Exec("INSERT INTO commit_status_summary (repo_id,sha,state,target_url) VALUES (?,?,?,?) ON DUPLICATE KEY UPDATE state=?",
67
+
repoID, sha, state.State, state.TargetURL, state.State)
68
+
return err
69
+
}
70
+
71
+
if cnt, err := db.GetEngine(ctx).Where("repo_id=? AND sha=?", repoID, sha).
72
+
Cols("state, target_url").
73
+
Update(&CommitStatusSummary{
74
+
State: state.State,
75
+
TargetURL: state.TargetURL,
76
+
}); err != nil {
77
+
return err
78
+
} else if cnt == 0 {
79
+
_, err = db.GetEngine(ctx).Insert(&CommitStatusSummary{
80
+
RepoID: repoID,
81
+
SHA: sha,
82
+
State: state.State,
83
+
TargetURL: state.TargetURL,
84
+
})
85
+
return err
86
+
}
87
+
return nil
88
+
}
+5
models/migrations/migrations.go
+5
models/migrations/migrations.go
···
578
578
579
579
// Gitea 1.22.0 ends at 294
580
580
581
+
// v294 -> v295
581
582
NewMigration("Add unique index for project issue table", v1_23.AddUniqueIndexForProjectIssue),
583
+
// v295 -> v296
584
+
NewMigration("Add commit status summary table", v1_23.AddCommitStatusSummary),
585
+
// v296 -> v297
586
+
NewMigration("Add missing field of commit status summary table", v1_23.AddCommitStatusSummary2),
582
587
}
583
588
584
589
// GetCurrentDBVersion returns the current db version
+18
models/migrations/v1_23/v295.go
+18
models/migrations/v1_23/v295.go
···
1
+
// Copyright 2024 The Gitea Authors. All rights reserved.
2
+
// SPDX-License-Identifier: MIT
3
+
4
+
package v1_23 //nolint
5
+
6
+
import "xorm.io/xorm"
7
+
8
+
func AddCommitStatusSummary(x *xorm.Engine) error {
9
+
type CommitStatusSummary struct {
10
+
ID int64 `xorm:"pk autoincr"`
11
+
RepoID int64 `xorm:"INDEX UNIQUE(repo_id_sha)"`
12
+
SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_id_sha)"`
13
+
State string `xorm:"VARCHAR(7) NOT NULL"`
14
+
}
15
+
// there is no migrations because if there is no data on this table, it will fall back to get data
16
+
// from commit status
17
+
return x.Sync2(new(CommitStatusSummary))
18
+
}
+16
models/migrations/v1_23/v296.go
+16
models/migrations/v1_23/v296.go
···
1
+
// Copyright 2024 The Gitea Authors. All rights reserved.
2
+
// SPDX-License-Identifier: MIT
3
+
4
+
package v1_23 //nolint
5
+
6
+
import "xorm.io/xorm"
7
+
8
+
func AddCommitStatusSummary2(x *xorm.Engine) error {
9
+
type CommitStatusSummary struct {
10
+
ID int64 `xorm:"pk autoincr"`
11
+
TargetURL string `xorm:"TEXT"`
12
+
}
13
+
// there is no migrations because if there is no data on this table, it will fall back to get data
14
+
// from commit status
15
+
return x.Sync(new(CommitStatusSummary))
16
+
}
+82
-18
services/repository/commitstatus/commitstatus.go
+82
-18
services/repository/commitstatus/commitstatus.go
···
7
7
"context"
8
8
"crypto/sha256"
9
9
"fmt"
10
+
"slices"
10
11
11
12
"code.gitea.io/gitea/models/db"
12
13
git_model "code.gitea.io/gitea/models/git"
···
15
16
"code.gitea.io/gitea/modules/cache"
16
17
"code.gitea.io/gitea/modules/git"
17
18
"code.gitea.io/gitea/modules/gitrepo"
19
+
"code.gitea.io/gitea/modules/json"
18
20
"code.gitea.io/gitea/modules/log"
19
21
api "code.gitea.io/gitea/modules/structs"
20
22
"code.gitea.io/gitea/services/automerge"
···
25
27
return fmt.Sprintf("commit_status:%x", hashBytes)
26
28
}
27
29
28
-
func updateCommitStatusCache(ctx context.Context, repoID int64, branchName string, status api.CommitStatusState) error {
30
+
type commitStatusCacheValue struct {
31
+
State string `json:"state"`
32
+
TargetURL string `json:"target_url"`
33
+
}
34
+
35
+
func getCommitStatusCache(repoID int64, branchName string) *commitStatusCacheValue {
29
36
c := cache.GetCache()
30
-
return c.Put(getCacheKey(repoID, branchName), string(status), 3*24*60)
37
+
statusStr, ok := c.Get(getCacheKey(repoID, branchName)).(string)
38
+
if ok && statusStr != "" {
39
+
var cv commitStatusCacheValue
40
+
err := json.Unmarshal([]byte(statusStr), &cv)
41
+
if err == nil && cv.State != "" {
42
+
return &cv
43
+
}
44
+
if err != nil {
45
+
log.Warn("getCommitStatusCache: json.Unmarshal failed: %v", err)
46
+
}
47
+
}
48
+
return nil
31
49
}
32
50
33
-
func deleteCommitStatusCache(ctx context.Context, repoID int64, branchName string) error {
51
+
func updateCommitStatusCache(repoID int64, branchName string, state api.CommitStatusState, targetURL string) error {
52
+
c := cache.GetCache()
53
+
bs, err := json.Marshal(commitStatusCacheValue{
54
+
State: state.String(),
55
+
TargetURL: targetURL,
56
+
})
57
+
if err != nil {
58
+
log.Warn("updateCommitStatusCache: json.Marshal failed: %v", err)
59
+
return nil
60
+
}
61
+
return c.Put(getCacheKey(repoID, branchName), string(bs), 3*24*60)
62
+
}
63
+
64
+
func deleteCommitStatusCache(repoID int64, branchName string) error {
34
65
c := cache.GetCache()
35
66
return c.Delete(getCacheKey(repoID, branchName))
36
67
}
···
59
90
sha = commit.ID.String()
60
91
}
61
92
62
-
if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
63
-
Repo: repo,
64
-
Creator: creator,
65
-
SHA: commit.ID,
66
-
CommitStatus: status,
93
+
if err := db.WithTx(ctx, func(ctx context.Context) error {
94
+
if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
95
+
Repo: repo,
96
+
Creator: creator,
97
+
SHA: commit.ID,
98
+
CommitStatus: status,
99
+
}); err != nil {
100
+
return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
101
+
}
102
+
103
+
return git_model.UpdateCommitStatusSummary(ctx, repo.ID, commit.ID.String())
67
104
}); err != nil {
68
-
return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
105
+
return err
69
106
}
70
107
71
108
defaultBranchCommit, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
···
74
111
}
75
112
76
113
if commit.ID.String() == defaultBranchCommit.ID.String() { // since one commit status updated, the combined commit status should be invalid
77
-
if err := deleteCommitStatusCache(ctx, repo.ID, repo.DefaultBranch); err != nil {
114
+
if err := deleteCommitStatusCache(repo.ID, repo.DefaultBranch); err != nil {
78
115
log.Error("deleteCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err)
79
116
}
80
117
}
···
91
128
// FindReposLastestCommitStatuses loading repository default branch latest combinded commit status with cache
92
129
func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Repository) ([]*git_model.CommitStatus, error) {
93
130
results := make([]*git_model.CommitStatus, len(repos))
94
-
c := cache.GetCache()
95
-
96
131
for i, repo := range repos {
97
-
status, ok := c.Get(getCacheKey(repo.ID, repo.DefaultBranch)).(string)
98
-
if ok && status != "" {
99
-
results[i] = &git_model.CommitStatus{State: api.CommitStatusState(status)}
132
+
if cv := getCommitStatusCache(repo.ID, repo.DefaultBranch); cv != nil {
133
+
results[i] = &git_model.CommitStatus{
134
+
State: api.CommitStatusState(cv.State),
135
+
TargetURL: cv.TargetURL,
136
+
}
100
137
}
101
138
}
102
139
···
114
151
return nil, fmt.Errorf("FindBranchesByRepoAndBranchName: %v", err)
115
152
}
116
153
154
+
var repoSHAs []git_model.RepoSHA
155
+
for id, sha := range repoIDsToLatestCommitSHAs {
156
+
repoSHAs = append(repoSHAs, git_model.RepoSHA{RepoID: id, SHA: sha})
157
+
}
158
+
159
+
summaryResults, err := git_model.GetLatestCommitStatusForRepoAndSHAs(ctx, repoSHAs)
160
+
if err != nil {
161
+
return nil, fmt.Errorf("GetLatestCommitStatusForRepoAndSHAs: %v", err)
162
+
}
163
+
164
+
for _, summary := range summaryResults {
165
+
for i, repo := range repos {
166
+
if repo.ID == summary.RepoID {
167
+
results[i] = summary
168
+
_ = slices.DeleteFunc(repoSHAs, func(repoSHA git_model.RepoSHA) bool {
169
+
return repoSHA.RepoID == repo.ID
170
+
})
171
+
if results[i].State != "" {
172
+
if err := updateCommitStatusCache(repo.ID, repo.DefaultBranch, results[i].State, results[i].TargetURL); err != nil {
173
+
log.Error("updateCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err)
174
+
}
175
+
}
176
+
break
177
+
}
178
+
}
179
+
}
180
+
117
181
// call the database O(1) times to get the commit statuses for all repos
118
-
repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptionsAll)
182
+
repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoSHAs)
119
183
if err != nil {
120
184
return nil, fmt.Errorf("GetLatestCommitStatusForPairs: %v", err)
121
185
}
···
123
187
for i, repo := range repos {
124
188
if results[i] == nil {
125
189
results[i] = git_model.CalcCommitStatus(repoToItsLatestCommitStatuses[repo.ID])
126
-
if results[i] != nil {
127
-
if err := updateCommitStatusCache(ctx, repo.ID, repo.DefaultBranch, results[i].State); err != nil {
190
+
if results[i] != nil && results[i].State != "" {
191
+
if err := updateCommitStatusCache(repo.ID, repo.DefaultBranch, results[i].State, results[i].TargetURL); err != nil {
128
192
log.Error("updateCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err)
129
193
}
130
194
}
+7
tests/integration/pull_status_test.go
+7
tests/integration/pull_status_test.go
···
12
12
"testing"
13
13
14
14
auth_model "code.gitea.io/gitea/models/auth"
15
+
git_model "code.gitea.io/gitea/models/git"
16
+
repo_model "code.gitea.io/gitea/models/repo"
17
+
"code.gitea.io/gitea/models/unittest"
15
18
api "code.gitea.io/gitea/modules/structs"
16
19
17
20
"github.com/stretchr/testify/assert"
···
90
93
assert.True(t, ok)
91
94
assert.Contains(t, cls, statesIcons[status])
92
95
}
96
+
97
+
repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: "user1", Name: "repo1"})
98
+
css := unittest.AssertExistsAndLoadBean(t, &git_model.CommitStatusSummary{RepoID: repo1.ID, SHA: commitID})
99
+
assert.EqualValues(t, api.CommitStatusWarning, css.State)
93
100
})
94
101
}
95
102