loading up the forgejo repo on tangled to test page performance
fix(migrations): transfer PR flow information (#7421)
- When migrating a pull requests from a Gitea or Forgejo instance, check if the pull request was created via the AGit flow and transfer that bit of information to the migrated pull request.
- Expose this bit of information as the `flow` field for the pull request.
- We have to do a horrible Go hack with Gitea's [go-sdk](gitea.com/gitea/go-sdk) to list all pull requests while being able to decode it to a struct that contains the new `Flow` field. The library does not allow you to do this out of the box, so we have to use `go:linkname` to access the private method that allows us to do this. This in turn means we have to do some boilerplate code that the library otherwise would do for us. The better option would be forking, but that would be a hassle of keeping the library in sync.
- Resolves forgejo/forgejo#5848
- Unit test added.
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/7421
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Gusted <postmaster@gusted.xyz>
Co-committed-by: Gusted <postmaster@gusted.xyz>
+131
-3
+1
modules/migration/pullrequest.go
+1
modules/migration/pullrequest.go
+2
-1
modules/structs/pull.go
+2
-1
modules/structs/pull.go
+1
services/convert/pull.go
+1
services/convert/pull.go
+39
-2
services/migrations/gitea_downloader.go
+39
-2
services/migrations/gitea_downloader.go
···
504
504
return allComments, true, nil
505
505
}
506
506
507
+
type ForgejoPullRequest struct {
508
+
gitea_sdk.PullRequest
509
+
Flow int64 `json:"flow"`
510
+
}
511
+
512
+
// Extracted from https://gitea.com/gitea/go-sdk/src/commit/164e3358bc02213954fb4380b821bed80a14824d/gitea/pull.go#L347-L364
513
+
func (g *GiteaDownloader) fixPullHeadSha(pr *ForgejoPullRequest) error {
514
+
if pr.Base != nil && pr.Base.Repository != nil && pr.Base.Repository.Owner != nil && pr.Head != nil && pr.Head.Ref != "" && pr.Head.Sha == "" {
515
+
owner := pr.Base.Repository.Owner.UserName
516
+
repo := pr.Base.Repository.Name
517
+
refs, _, err := g.client.GetRepoRefs(owner, repo, pr.Head.Ref)
518
+
if err != nil {
519
+
return err
520
+
}
521
+
if len(refs) == 0 {
522
+
return fmt.Errorf("unable to resolve PR ref %q", pr.Head.Ref)
523
+
}
524
+
pr.Head.Sha = refs[0].Object.SHA
525
+
}
526
+
return nil
527
+
}
528
+
507
529
// GetPullRequests returns pull requests according page and perPage
508
530
func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
509
531
if perPage > g.maxPerPage {
···
511
533
}
512
534
allPRs := make([]*base.PullRequest, 0, perPage)
513
535
514
-
prs, _, err := g.client.ListRepoPullRequests(g.repoOwner, g.repoName, gitea_sdk.ListPullRequestsOptions{
536
+
prs := make([]*ForgejoPullRequest, 0, perPage)
537
+
opt := gitea_sdk.ListPullRequestsOptions{
515
538
ListOptions: gitea_sdk.ListOptions{
516
539
Page: page,
517
540
PageSize: perPage,
518
541
},
519
542
State: gitea_sdk.StateAll,
520
-
})
543
+
}
544
+
545
+
link, _ := url.Parse(fmt.Sprintf("/repos/%s/%s/pulls", url.PathEscape(g.repoOwner), url.PathEscape(g.repoName)))
546
+
link.RawQuery = opt.QueryEncode()
547
+
_, err := getParsedResponse(g.client, "GET", link.String(), http.Header{"content-type": []string{"application/json"}}, nil, &prs)
521
548
if err != nil {
522
549
return nil, false, fmt.Errorf("error while listing pull requests (page: %d, pagesize: %d). Error: %w", page, perPage, err)
523
550
}
551
+
552
+
if g.client.CheckServerVersionConstraint(">= 1.14.0") != nil {
553
+
for i := range prs {
554
+
if err := g.fixPullHeadSha(prs[i]); err != nil {
555
+
return nil, false, fmt.Errorf("error while listing pull requests (page: %d, pagesize: %d). Error: %w", page, perPage, err)
556
+
}
557
+
}
558
+
}
559
+
524
560
for _, pr := range prs {
525
561
var milestone string
526
562
if pr.Milestone != nil {
···
598
634
MergeCommitSHA: mergeCommitSHA,
599
635
IsLocked: pr.IsLocked,
600
636
PatchURL: pr.PatchURL,
637
+
Flow: pr.Flow,
601
638
Head: base.PullRequestBranch{
602
639
Ref: headRef,
603
640
SHA: headSHA,
+43
services/migrations/gitea_downloader_test.go
+43
services/migrations/gitea_downloader_test.go
···
307
307
},
308
308
}, reviews)
309
309
}
310
+
311
+
func TestForgejoDownloadRepo(t *testing.T) {
312
+
token := os.Getenv("CODE_FORGEJO_TOKEN")
313
+
314
+
fixturePath := "./testdata/code-forgejo-org/full_download"
315
+
server := unittest.NewMockWebServer(t, "https://code.forgejo.org", fixturePath, token != "")
316
+
defer server.Close()
317
+
318
+
downloader, err := NewGiteaDownloader(t.Context(), server.URL, "Gusted/agit-test", "", "", token)
319
+
require.NoError(t, err)
320
+
require.NotNil(t, downloader)
321
+
322
+
prs, _, err := downloader.GetPullRequests(1, 50)
323
+
require.NoError(t, err)
324
+
assert.Len(t, prs, 1)
325
+
326
+
assertPullRequestEqual(t, &base.PullRequest{
327
+
Number: 1,
328
+
PosterID: 63,
329
+
PosterName: "Gusted",
330
+
PosterEmail: "postmaster@gusted.xyz",
331
+
Title: "Add extra information",
332
+
State: "open",
333
+
Created: time.Date(2025, time.April, 1, 20, 28, 45, 0, time.UTC),
334
+
Updated: time.Date(2025, time.April, 1, 20, 28, 45, 0, time.UTC),
335
+
Base: base.PullRequestBranch{
336
+
CloneURL: "",
337
+
Ref: "main",
338
+
SHA: "79ebb873a6497c8847141ba9706b3f757196a1e6",
339
+
RepoName: "agit-test",
340
+
OwnerName: "Gusted",
341
+
},
342
+
Head: base.PullRequestBranch{
343
+
CloneURL: server.URL + "/Gusted/agit-test.git",
344
+
Ref: "refs/pull/1/head",
345
+
SHA: "667e9317ec37b977e6d3d7d43e3440636970563c",
346
+
RepoName: "agit-test",
347
+
OwnerName: "Gusted",
348
+
},
349
+
PatchURL: server.URL + "/Gusted/agit-test/pulls/1.patch",
350
+
Flow: 1,
351
+
}, prs[0])
352
+
}
+16
services/migrations/gitea_sdk_hack.go
+16
services/migrations/gitea_sdk_hack.go
···
1
+
// Copyright 2025 The Forgejo Authors. All rights reserved.
2
+
// SPDX-License-Identifier: GPL-3.0-or-later
3
+
4
+
package migrations
5
+
6
+
import (
7
+
"io"
8
+
"net/http"
9
+
10
+
_ "unsafe" // Needed for go:linkname support
11
+
12
+
gitea_sdk "code.gitea.io/sdk/gitea"
13
+
)
14
+
15
+
//go:linkname getParsedResponse code.gitea.io/sdk/gitea.(*Client).getParsedResponse
16
+
func getParsedResponse(client *gitea_sdk.Client, method, path string, header http.Header, body io.Reader, obj any) (*gitea_sdk.Response, error)
+1
services/migrations/gitea_uploader.go
+1
services/migrations/gitea_uploader.go
+1
services/migrations/main_test.go
+1
services/migrations/main_test.go
···
136
136
assert.ElementsMatch(t, expected.Assignees, actual.Assignees)
137
137
assert.Equal(t, expected.IsLocked, actual.IsLocked)
138
138
assertReactionsEqual(t, expected.Reactions, actual.Reactions)
139
+
assert.Equal(t, expected.Flow, actual.Flow)
139
140
}
140
141
141
142
func assertPullRequestsEqual(t *testing.T, expected, actual []*base.PullRequest) {
+8
services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Frepos%2FGusted%2Fagit-test%2Fpulls%3Flimit=50&page=1&state=all
+8
services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Frepos%2FGusted%2Fagit-test%2Fpulls%3Flimit=50&page=1&state=all
···
1
+
Access-Control-Expose-Headers: X-Total-Count
2
+
Cache-Control: max-age=0, private, must-revalidate, no-transform
3
+
Content-Type: application/json;charset=utf-8
4
+
X-Content-Type-Options: nosniff
5
+
X-Frame-Options: SAMEORIGIN
6
+
X-Total-Count: 1
7
+
8
+
[{"id":4980,"url":"https://code.forgejo.org/Gusted/agit-test/pulls/1","number":1,"user":{"id":63,"login":"Gusted","login_name":"26734","source_id":1,"full_name":"","email":"postmaster@gusted.xyz","avatar_url":"https://code.forgejo.org/avatars/4ca5ad8bc488630869fdbd2051da61cbed7241c9c066d4e5e1dd36300f887340","html_url":"https://code.forgejo.org/Gusted","language":"en-US","is_admin":false,"last_login":"2025-04-01T16:35:18Z","created":"2023-07-08T13:33:38Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":2,"following_count":0,"starred_repos_count":0,"username":"Gusted"},"title":"Add extra information","body":"","labels":[],"milestone":null,"assignee":null,"assignees":null,"requested_reviewers":[],"requested_reviewers_teams":[],"state":"open","draft":false,"is_locked":false,"comments":0,"review_comments":0,"additions":0,"deletions":0,"changed_files":0,"html_url":"https://code.forgejo.org/Gusted/agit-test/pulls/1","diff_url":"https://code.forgejo.org/Gusted/agit-test/pulls/1.diff","patch_url":"https://code.forgejo.org/Gusted/agit-test/pulls/1.patch","mergeable":true,"merged":false,"merged_at":null,"merge_commit_sha":null,"merged_by":null,"allow_maintainer_edit":false,"base":{"label":"main","ref":"main","sha":"79ebb873a6497c8847141ba9706b3f757196a1e6","repo_id":1414,"repo":{"id":1414,"owner":{"id":63,"login":"Gusted","login_name":"","source_id":0,"full_name":"","email":"gusted@noreply.code.forgejo.org","avatar_url":"https://code.forgejo.org/avatars/4ca5ad8bc488630869fdbd2051da61cbed7241c9c066d4e5e1dd36300f887340","html_url":"https://code.forgejo.org/Gusted","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2023-07-08T13:33:38Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":2,"following_count":0,"starred_repos_count":0,"username":"Gusted"},"name":"agit-test","full_name":"Gusted/agit-test","description":"USED FOR FORGEJO UNIT TESTING","empty":false,"private":false,"fork":false,"template":false,"parent":null,"mirror":false,"size":36,"language":"","languages_url":"https://code.forgejo.org/api/v1/repos/Gusted/agit-test/languages","html_url":"https://code.forgejo.org/Gusted/agit-test","url":"https://code.forgejo.org/api/v1/repos/Gusted/agit-test","link":"","ssh_url":"ssh://git@code.forgejo.org/Gusted/agit-test.git","clone_url":"https://code.forgejo.org/Gusted/agit-test.git","original_url":"","website":"","stars_count":0,"forks_count":0,"watchers_count":1,"open_issues_count":0,"open_pr_counter":1,"release_counter":0,"default_branch":"main","archived":false,"created_at":"2025-04-01T20:25:03Z","updated_at":"2025-04-01T20:25:03Z","archived_at":"1970-01-01T00:00:00Z","permissions":{"admin":true,"push":true,"pull":true},"has_issues":true,"internal_tracker":{"enable_time_tracker":true,"allow_only_contributors_to_track_time":true,"enable_issue_dependencies":true},"has_wiki":true,"wiki_branch":"main","globally_editable_wiki":false,"has_pull_requests":true,"has_projects":true,"has_releases":true,"has_packages":true,"has_actions":true,"ignore_whitespace_conflicts":false,"allow_merge_commits":true,"allow_rebase":true,"allow_rebase_explicit":true,"allow_squash_merge":true,"allow_fast_forward_only_merge":true,"allow_rebase_update":true,"default_delete_branch_after_merge":false,"default_merge_style":"merge","default_allow_maintainer_edit":false,"default_update_style":"merge","avatar_url":"","internal":false,"mirror_interval":"","object_format_name":"sha1","mirror_updated":"0001-01-01T00:00:00Z","repo_transfer":null,"topics":null}},"head":{"label":"","ref":"refs/pull/1/head","sha":"667e9317ec37b977e6d3d7d43e3440636970563c","repo_id":1414,"repo":{"id":1414,"owner":{"id":63,"login":"Gusted","login_name":"","source_id":0,"full_name":"","email":"gusted@noreply.code.forgejo.org","avatar_url":"https://code.forgejo.org/avatars/4ca5ad8bc488630869fdbd2051da61cbed7241c9c066d4e5e1dd36300f887340","html_url":"https://code.forgejo.org/Gusted","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2023-07-08T13:33:38Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":2,"following_count":0,"starred_repos_count":0,"username":"Gusted"},"name":"agit-test","full_name":"Gusted/agit-test","description":"USED FOR FORGEJO UNIT TESTING","empty":false,"private":false,"fork":false,"template":false,"parent":null,"mirror":false,"size":36,"language":"","languages_url":"https://code.forgejo.org/api/v1/repos/Gusted/agit-test/languages","html_url":"https://code.forgejo.org/Gusted/agit-test","url":"https://code.forgejo.org/api/v1/repos/Gusted/agit-test","link":"","ssh_url":"ssh://git@code.forgejo.org/Gusted/agit-test.git","clone_url":"https://code.forgejo.org/Gusted/agit-test.git","original_url":"","website":"","stars_count":0,"forks_count":0,"watchers_count":1,"open_issues_count":0,"open_pr_counter":1,"release_counter":0,"default_branch":"main","archived":false,"created_at":"2025-04-01T20:25:03Z","updated_at":"2025-04-01T20:25:03Z","archived_at":"1970-01-01T00:00:00Z","permissions":{"admin":true,"push":true,"pull":true},"has_issues":true,"internal_tracker":{"enable_time_tracker":true,"allow_only_contributors_to_track_time":true,"enable_issue_dependencies":true},"has_wiki":true,"wiki_branch":"main","globally_editable_wiki":false,"has_pull_requests":true,"has_projects":true,"has_releases":true,"has_packages":true,"has_actions":true,"ignore_whitespace_conflicts":false,"allow_merge_commits":true,"allow_rebase":true,"allow_rebase_explicit":true,"allow_squash_merge":true,"allow_fast_forward_only_merge":true,"allow_rebase_update":true,"default_delete_branch_after_merge":false,"default_merge_style":"merge","default_allow_maintainer_edit":false,"default_update_style":"merge","avatar_url":"","internal":false,"mirror_interval":"","object_format_name":"sha1","mirror_updated":"0001-01-01T00:00:00Z","repo_transfer":null,"topics":null}},"merge_base":"79ebb873a6497c8847141ba9706b3f757196a1e6","due_date":null,"created_at":"2025-04-01T20:28:45Z","updated_at":"2025-04-01T20:28:45Z","closed_at":null,"pin_order":0,"flow":1}]
+7
services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Fsettings%2Fapi
+7
services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Fsettings%2Fapi
···
1
+
Content-Length: 117
2
+
Cache-Control: max-age=0, private, must-revalidate, no-transform
3
+
Content-Type: application/json;charset=utf-8
4
+
X-Content-Type-Options: nosniff
5
+
X-Frame-Options: SAMEORIGIN
6
+
7
+
{"max_response_items":50,"default_paging_num":30,"default_git_trees_per_page":1000,"default_max_blob_size":10485760}
+7
services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Fversion
+7
services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Fversion
+5
templates/swagger/v1_json.tmpl
+5
templates/swagger/v1_json.tmpl