loading up the forgejo repo on tangled to test page performance
Implement sync push mirror on commit (#19411)
Support synchronizing with the push mirrors whenever new commits are pushed or synced from pull mirror.
Related Issues: #18220
Co-authored-by: delvh <dev.lh@web.de>
Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
authored by
Chongyi Zheng
delvh
zeripath
Lunny Xiao
and committed by
GitHub
49f9d43a
496b8e39
+208
-98
+2
models/migrations/migrations.go
+2
models/migrations/migrations.go
···
396
396
NewMigration("Alter hook_task table TEXT fields to LONGTEXT", alterHookTaskTextFieldsToLongText),
397
397
// v218 -> v219
398
398
NewMigration("Improve Action table indices v2", improveActionTableIndices),
399
+
// v219 -> v220
400
+
NewMigration("Add sync_on_commit column to push_mirror table", addSyncOnCommitColForPushMirror),
399
401
}
400
402
401
403
// GetCurrentDBVersion returns the current db version
+30
models/migrations/v219.go
+30
models/migrations/v219.go
···
1
+
// Copyright 2022 The Gitea Authors. All rights reserved.
2
+
// Use of this source code is governed by a MIT-style
3
+
// license that can be found in the LICENSE file.
4
+
5
+
package migrations
6
+
7
+
import (
8
+
"time"
9
+
10
+
"code.gitea.io/gitea/models/repo"
11
+
"code.gitea.io/gitea/modules/timeutil"
12
+
"xorm.io/xorm"
13
+
)
14
+
15
+
func addSyncOnCommitColForPushMirror(x *xorm.Engine) error {
16
+
type PushMirror struct {
17
+
ID int64 `xorm:"pk autoincr"`
18
+
RepoID int64 `xorm:"INDEX"`
19
+
Repo *repo.Repository `xorm:"-"`
20
+
RemoteName string
21
+
22
+
SyncOnCommit bool `xorm:"NOT NULL DEFAULT true"`
23
+
Interval time.Duration
24
+
CreatedUnix timeutil.TimeStamp `xorm:"created"`
25
+
LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
26
+
LastError string `xorm:"text"`
27
+
}
28
+
29
+
return x.Sync2(new(PushMirror))
30
+
}
+9
models/repo/pushmirror.go
+9
models/repo/pushmirror.go
···
23
23
Repo *Repository `xorm:"-"`
24
24
RemoteName string
25
25
26
+
SyncOnCommit bool `xorm:"NOT NULL DEFAULT true"`
26
27
Interval time.Duration
27
28
CreatedUnix timeutil.TimeStamp `xorm:"created"`
28
29
LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
···
91
92
func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) {
92
93
mirrors := make([]*PushMirror, 0, 10)
93
94
return mirrors, db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID).Find(&mirrors)
95
+
}
96
+
97
+
// GetPushMirrorsSyncedOnCommit returns push-mirrors for this repo that should be updated by new commits
98
+
func GetPushMirrorsSyncedOnCommit(repoID int64) ([]*PushMirror, error) {
99
+
mirrors := make([]*PushMirror, 0, 10)
100
+
return mirrors, db.GetEngine(db.DefaultContext).
101
+
Where("repo_id=? AND sync_on_commit=?", repoID, true).
102
+
Find(&mirrors)
94
103
}
95
104
96
105
// PushMirrorsIterate iterates all push-mirror repositories.
+69
modules/mirror/mirror.go
+69
modules/mirror/mirror.go
···
1
+
// Copyright 2022 The Gitea Authors. All rights reserved.
2
+
// Use of this source code is governed by a MIT-style
3
+
// license that can be found in the LICENSE file.
4
+
5
+
package mirror
6
+
7
+
import (
8
+
"code.gitea.io/gitea/modules/graceful"
9
+
"code.gitea.io/gitea/modules/log"
10
+
"code.gitea.io/gitea/modules/queue"
11
+
"code.gitea.io/gitea/modules/setting"
12
+
)
13
+
14
+
var mirrorQueue queue.UniqueQueue
15
+
16
+
// SyncType type of sync request
17
+
type SyncType int
18
+
19
+
const (
20
+
// PullMirrorType for pull mirrors
21
+
PullMirrorType SyncType = iota
22
+
// PushMirrorType for push mirrors
23
+
PushMirrorType
24
+
)
25
+
26
+
// SyncRequest for the mirror queue
27
+
type SyncRequest struct {
28
+
Type SyncType
29
+
ReferenceID int64 // RepoID for pull mirror, MirrorID for push mirror
30
+
}
31
+
32
+
// StartSyncMirrors starts a go routine to sync the mirrors
33
+
func StartSyncMirrors(queueHandle func(data ...queue.Data) []queue.Data) {
34
+
if !setting.Mirror.Enabled {
35
+
return
36
+
}
37
+
mirrorQueue = queue.CreateUniqueQueue("mirror", queueHandle, new(SyncRequest))
38
+
39
+
go graceful.GetManager().RunWithShutdownFns(mirrorQueue.Run)
40
+
}
41
+
42
+
// AddPullMirrorToQueue adds repoID to mirror queue
43
+
func AddPullMirrorToQueue(repoID int64) {
44
+
addMirrorToQueue(PullMirrorType, repoID)
45
+
}
46
+
47
+
// AddPushMirrorToQueue adds the push mirror to the queue
48
+
func AddPushMirrorToQueue(mirrorID int64) {
49
+
addMirrorToQueue(PushMirrorType, mirrorID)
50
+
}
51
+
52
+
func addMirrorToQueue(syncType SyncType, referenceID int64) {
53
+
if !setting.Mirror.Enabled {
54
+
return
55
+
}
56
+
go func() {
57
+
if err := PushToQueue(syncType, referenceID); err != nil {
58
+
log.Error("Unable to push sync request for to the queue for pull mirror repo[%d]. Error: %v", referenceID, err)
59
+
}
60
+
}()
61
+
}
62
+
63
+
// PushToQueue adds the sync request to the queue
64
+
func PushToQueue(mirrorType SyncType, referenceID int64) error {
65
+
return mirrorQueue.Push(&SyncRequest{
66
+
Type: mirrorType,
67
+
ReferenceID: referenceID,
68
+
})
69
+
}
+45
modules/notification/mirror/mirror.go
+45
modules/notification/mirror/mirror.go
···
1
+
// Copyright 2022 The Gitea Authors. All rights reserved.
2
+
// Use of this source code is governed by a MIT-style
3
+
// license that can be found in the LICENSE file.
4
+
5
+
package mirror
6
+
7
+
import (
8
+
repo_model "code.gitea.io/gitea/models/repo"
9
+
user_model "code.gitea.io/gitea/models/user"
10
+
"code.gitea.io/gitea/modules/log"
11
+
mirror_module "code.gitea.io/gitea/modules/mirror"
12
+
"code.gitea.io/gitea/modules/notification/base"
13
+
"code.gitea.io/gitea/modules/repository"
14
+
)
15
+
16
+
type mirrorNotifier struct {
17
+
base.NullNotifier
18
+
}
19
+
20
+
var _ base.Notifier = &mirrorNotifier{}
21
+
22
+
// NewNotifier create a new mirrorNotifier notifier
23
+
func NewNotifier() base.Notifier {
24
+
return &mirrorNotifier{}
25
+
}
26
+
27
+
func (m *mirrorNotifier) NotifyPushCommits(_ *user_model.User, repo *repo_model.Repository, _ *repository.PushUpdateOptions, _ *repository.PushCommits) {
28
+
syncPushMirrorWithSyncOnCommit(repo.ID)
29
+
}
30
+
31
+
func (m *mirrorNotifier) NotifySyncPushCommits(_ *user_model.User, repo *repo_model.Repository, _ *repository.PushUpdateOptions, _ *repository.PushCommits) {
32
+
syncPushMirrorWithSyncOnCommit(repo.ID)
33
+
}
34
+
35
+
func syncPushMirrorWithSyncOnCommit(repoID int64) {
36
+
pushMirrors, err := repo_model.GetPushMirrorsSyncedOnCommit(repoID)
37
+
if err != nil {
38
+
log.Error("repo_model.GetPushMirrorsSyncedOnCommit failed: %v", err)
39
+
return
40
+
}
41
+
42
+
for _, mirror := range pushMirrors {
43
+
mirror_module.AddPushMirrorToQueue(mirror.ID)
44
+
}
45
+
}
+2
modules/notification/notification.go
+2
modules/notification/notification.go
···
14
14
"code.gitea.io/gitea/modules/notification/base"
15
15
"code.gitea.io/gitea/modules/notification/indexer"
16
16
"code.gitea.io/gitea/modules/notification/mail"
17
+
"code.gitea.io/gitea/modules/notification/mirror"
17
18
"code.gitea.io/gitea/modules/notification/ui"
18
19
"code.gitea.io/gitea/modules/notification/webhook"
19
20
"code.gitea.io/gitea/modules/repository"
···
37
38
RegisterNotifier(indexer.NewNotifier())
38
39
RegisterNotifier(webhook.NewNotifier())
39
40
RegisterNotifier(action.NewNotifier())
41
+
RegisterNotifier(mirror.NewNotifier())
40
42
}
41
43
42
44
// NotifyCreateIssueComment notifies issue comment related message to notifiers
+2
-1
options/locale/locale_en-US.ini
+2
-1
options/locale/locale_en-US.ini
···
861
861
default_branch_helper = The default branch is the base branch for pull requests and code commits.
862
862
mirror_prune = Prune
863
863
mirror_prune_desc = Remove obsolete remote-tracking references
864
-
mirror_interval = Mirror Interval (valid time units are 'h', 'm', 's'). 0 to disable automatic sync. (Minimum interval: %s)
864
+
mirror_interval = Mirror Interval (valid time units are 'h', 'm', 's'). 0 to disable periodic sync. (Minimum interval: %s)
865
865
mirror_interval_invalid = The mirror interval is not valid.
866
+
mirror_sync_on_commit = Sync when commits are pushed
866
867
mirror_address = Clone From URL
867
868
mirror_address_desc = Put any required credentials in the Authorization section.
868
869
mirror_address_url_invalid = The provided url is invalid. You must escape all components of the url correctly.
+2
-2
routers/api/v1/repo/mirror.go
+2
-2
routers/api/v1/repo/mirror.go
···
11
11
repo_model "code.gitea.io/gitea/models/repo"
12
12
"code.gitea.io/gitea/models/unit"
13
13
"code.gitea.io/gitea/modules/context"
14
+
mirror_module "code.gitea.io/gitea/modules/mirror"
14
15
"code.gitea.io/gitea/modules/setting"
15
-
mirror_service "code.gitea.io/gitea/services/mirror"
16
16
)
17
17
18
18
// MirrorSync adds a mirrored repository to the sync queue
···
59
59
return
60
60
}
61
61
62
-
mirror_service.StartToMirror(repo.ID)
62
+
mirror_module.AddPullMirrorToQueue(repo.ID)
63
63
64
64
ctx.Status(http.StatusOK)
65
65
}
+8
-6
routers/web/repo/setting.go
+8
-6
routers/web/repo/setting.go
···
29
29
"code.gitea.io/gitea/modules/indexer/stats"
30
30
"code.gitea.io/gitea/modules/lfs"
31
31
"code.gitea.io/gitea/modules/log"
32
+
mirror_module "code.gitea.io/gitea/modules/mirror"
32
33
"code.gitea.io/gitea/modules/repository"
33
34
"code.gitea.io/gitea/modules/setting"
34
35
"code.gitea.io/gitea/modules/structs"
···
272
273
return
273
274
}
274
275
275
-
mirror_service.StartToMirror(repo.ID)
276
+
mirror_module.AddPullMirrorToQueue(repo.ID)
276
277
277
278
ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress"))
278
279
ctx.Redirect(repo.Link() + "/settings")
···
289
290
return
290
291
}
291
292
292
-
mirror_service.AddPushMirrorToQueue(m.ID)
293
+
mirror_module.AddPushMirrorToQueue(m.ID)
293
294
294
295
ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress"))
295
296
ctx.Redirect(repo.Link() + "/settings")
···
357
358
}
358
359
359
360
m := &repo_model.PushMirror{
360
-
RepoID: repo.ID,
361
-
Repo: repo,
362
-
RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix),
363
-
Interval: interval,
361
+
RepoID: repo.ID,
362
+
Repo: repo,
363
+
RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix),
364
+
SyncOnCommit: form.PushMirrorSyncOnCommit,
365
+
Interval: interval,
364
366
}
365
367
if err := repo_model.InsertPushMirror(m); err != nil {
366
368
ctx.ServerError("InsertPushMirror", err)
+18
-17
services/forms/repo_form.go
+18
-17
services/forms/repo_form.go
···
115
115
116
116
// RepoSettingForm form for changing repository settings
117
117
type RepoSettingForm struct {
118
-
RepoName string `binding:"Required;AlphaDashDot;MaxSize(100)"`
119
-
Description string `binding:"MaxSize(255)"`
120
-
Website string `binding:"ValidUrl;MaxSize(255)"`
121
-
Interval string
122
-
MirrorAddress string
123
-
MirrorUsername string
124
-
MirrorPassword string
125
-
LFS bool `form:"mirror_lfs"`
126
-
LFSEndpoint string `form:"mirror_lfs_endpoint"`
127
-
PushMirrorID string
128
-
PushMirrorAddress string
129
-
PushMirrorUsername string
130
-
PushMirrorPassword string
131
-
PushMirrorInterval string
132
-
Private bool
133
-
Template bool
134
-
EnablePrune bool
118
+
RepoName string `binding:"Required;AlphaDashDot;MaxSize(100)"`
119
+
Description string `binding:"MaxSize(255)"`
120
+
Website string `binding:"ValidUrl;MaxSize(255)"`
121
+
Interval string
122
+
MirrorAddress string
123
+
MirrorUsername string
124
+
MirrorPassword string
125
+
LFS bool `form:"mirror_lfs"`
126
+
LFSEndpoint string `form:"mirror_lfs_endpoint"`
127
+
PushMirrorID string
128
+
PushMirrorAddress string
129
+
PushMirrorUsername string
130
+
PushMirrorPassword string
131
+
PushMirrorSyncOnCommit bool
132
+
PushMirrorInterval string
133
+
Private bool
134
+
Template bool
135
+
EnablePrune bool
135
136
136
137
// Advanced settings
137
138
EnableWiki bool
+15
-72
services/mirror/mirror.go
+15
-72
services/mirror/mirror.go
···
11
11
repo_model "code.gitea.io/gitea/models/repo"
12
12
"code.gitea.io/gitea/modules/graceful"
13
13
"code.gitea.io/gitea/modules/log"
14
+
mirror_module "code.gitea.io/gitea/modules/mirror"
14
15
"code.gitea.io/gitea/modules/queue"
15
16
"code.gitea.io/gitea/modules/setting"
16
17
)
17
18
18
-
var mirrorQueue queue.UniqueQueue
19
-
20
-
// SyncType type of sync request
21
-
type SyncType int
22
-
23
-
const (
24
-
// PullMirrorType for pull mirrors
25
-
PullMirrorType SyncType = iota
26
-
// PushMirrorType for push mirrors
27
-
PushMirrorType
28
-
)
29
-
30
-
// SyncRequest for the mirror queue
31
-
type SyncRequest struct {
32
-
Type SyncType
33
-
ReferenceID int64 // RepoID for pull mirror, MirrorID fro push mirror
34
-
}
35
-
36
19
// doMirrorSync causes this request to mirror itself
37
-
func doMirrorSync(ctx context.Context, req *SyncRequest) {
20
+
func doMirrorSync(ctx context.Context, req *mirror_module.SyncRequest) {
38
21
if req.ReferenceID == 0 {
39
22
log.Warn("Skipping mirror sync request, no mirror ID was specified")
40
23
return
41
24
}
42
25
switch req.Type {
43
-
case PushMirrorType:
26
+
case mirror_module.PushMirrorType:
44
27
_ = SyncPushMirror(ctx, req.ReferenceID)
45
-
case PullMirrorType:
28
+
case mirror_module.PullMirrorType:
46
29
_ = SyncPullMirror(ctx, req.ReferenceID)
47
30
default:
48
31
log.Error("Unknown Request type in queue: %v for MirrorID[%d]", req.Type, req.ReferenceID)
···
60
43
log.Trace("Doing: Update")
61
44
62
45
handler := func(idx int, bean interface{}) error {
63
-
var item SyncRequest
64
46
var repo *repo_model.Repository
47
+
var mirrorType mirror_module.SyncType
48
+
var referenceID int64
49
+
65
50
if m, ok := bean.(*repo_model.Mirror); ok {
66
51
if m.GetRepository() == nil {
67
52
log.Error("Disconnected mirror found: %d", m.ID)
68
53
return nil
69
54
}
70
55
repo = m.Repo
71
-
item = SyncRequest{
72
-
Type: PullMirrorType,
73
-
ReferenceID: m.RepoID,
74
-
}
56
+
mirrorType = mirror_module.PullMirrorType
57
+
referenceID = m.RepoID
75
58
} else if m, ok := bean.(*repo_model.PushMirror); ok {
76
59
if m.GetRepository() == nil {
77
60
log.Error("Disconnected push-mirror found: %d", m.ID)
78
61
return nil
79
62
}
80
63
repo = m.Repo
81
-
item = SyncRequest{
82
-
Type: PushMirrorType,
83
-
ReferenceID: m.ID,
84
-
}
64
+
mirrorType = mirror_module.PushMirrorType
65
+
referenceID = m.ID
85
66
} else {
86
67
log.Error("Unknown bean: %v", bean)
87
68
return nil
···
95
76
}
96
77
97
78
// Push to the Queue
98
-
if err := mirrorQueue.Push(&item); err != nil {
79
+
if err := mirror_module.PushToQueue(mirrorType, referenceID); err != nil {
99
80
if err == queue.ErrAlreadyInQueue {
100
-
if item.Type == PushMirrorType {
81
+
if mirrorType == mirror_module.PushMirrorType {
101
82
log.Trace("PushMirrors for %-v already queued for sync", repo)
102
83
} else {
103
84
log.Trace("PullMirrors for %-v already queued for sync", repo)
···
142
123
143
124
func queueHandle(data ...queue.Data) []queue.Data {
144
125
for _, datum := range data {
145
-
req := datum.(*SyncRequest)
126
+
req := datum.(*mirror_module.SyncRequest)
146
127
doMirrorSync(graceful.GetManager().ShutdownContext(), req)
147
128
}
148
129
return nil
···
150
131
151
132
// InitSyncMirrors initializes a go routine to sync the mirrors
152
133
func InitSyncMirrors() {
153
-
if !setting.Mirror.Enabled {
154
-
return
155
-
}
156
-
mirrorQueue = queue.CreateUniqueQueue("mirror", queueHandle, new(SyncRequest))
157
-
158
-
go graceful.GetManager().RunWithShutdownFns(mirrorQueue.Run)
159
-
}
160
-
161
-
// StartToMirror adds repoID to mirror queue
162
-
func StartToMirror(repoID int64) {
163
-
if !setting.Mirror.Enabled {
164
-
return
165
-
}
166
-
go func() {
167
-
err := mirrorQueue.Push(&SyncRequest{
168
-
Type: PullMirrorType,
169
-
ReferenceID: repoID,
170
-
})
171
-
if err != nil {
172
-
log.Error("Unable to push sync request for to the queue for pull mirror repo[%d]: Error: %v", repoID, err)
173
-
return
174
-
}
175
-
}()
176
-
}
177
-
178
-
// AddPushMirrorToQueue adds the push mirror to the queue
179
-
func AddPushMirrorToQueue(mirrorID int64) {
180
-
if !setting.Mirror.Enabled {
181
-
return
182
-
}
183
-
go func() {
184
-
err := mirrorQueue.Push(&SyncRequest{
185
-
Type: PushMirrorType,
186
-
ReferenceID: mirrorID,
187
-
})
188
-
if err != nil {
189
-
log.Error("Unable to push sync request to the queue for pull mirror repo[%d]: Error: %v", mirrorID, err)
190
-
}
191
-
}()
134
+
mirror_module.StartSyncMirrors(queueHandle)
192
135
}
+6
templates/repo/settings/options.tmpl
+6
templates/repo/settings/options.tmpl
···
219
219
</div>
220
220
</div>
221
221
</details>
222
+
<div class="field">
223
+
<div class="ui checkbox">
224
+
<input id="push_mirror_sync_on_commit" name="push_mirror_sync_on_commit" type="checkbox" value="{{.push_mirror_sync_on_commit}}">
225
+
<label for="push_mirror_sync_on_commit">{{.locale.Tr "repo.mirror_sync_on_commit"}}</label>
226
+
</div>
227
+
</div>
222
228
<div class="inline field {{if .Err_PushMirrorInterval}}error{{end}}">
223
229
<label for="push_mirror_interval">{{.locale.Tr "repo.mirror_interval" .MinimumMirrorInterval}}</label>
224
230
<input id="push_mirror_interval" name="push_mirror_interval" value="{{if .push_mirror_interval}}{{.push_mirror_interval}}{{else}}{{.DefaultMirrorInterval}}{{end}}">