loading up the forgejo repo on tangled to test page performance
Merge pull request 'fix: use buffered iterate for debian searchpackages' (#5702) from gusted/forgejo-buffered-iterate into forgejo
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/5702
Reviewed-by: Otto <otto@codeberg.org>
+124
-14
+31
-14
models/packages/debian/search.go
+31
-14
models/packages/debian/search.go
···
10
10
"code.gitea.io/gitea/models/db"
11
11
"code.gitea.io/gitea/models/packages"
12
12
debian_module "code.gitea.io/gitea/modules/packages/debian"
13
+
"code.gitea.io/gitea/modules/setting"
13
14
14
15
"xorm.io/builder"
15
16
)
···
76
77
77
78
// SearchPackages gets the packages matching the search options
78
79
func SearchPackages(ctx context.Context, opts *PackageSearchOptions, iter func(*packages.PackageFileDescriptor)) error {
79
-
return db.GetEngine(ctx).
80
-
Table("package_file").
81
-
Select("package_file.*").
82
-
Join("INNER", "package_version", "package_version.id = package_file.version_id").
83
-
Join("INNER", "package", "package.id = package_version.package_id").
84
-
Where(opts.toCond()).
85
-
Asc("package.lower_name", "package_version.created_unix").
86
-
Iterate(new(packages.PackageFile), func(_ int, bean any) error {
87
-
pf := bean.(*packages.PackageFile)
80
+
var start int
81
+
batchSize := setting.Database.IterateBufferSize
82
+
for {
83
+
select {
84
+
case <-ctx.Done():
85
+
return ctx.Err()
86
+
default:
87
+
beans := make([]*packages.PackageFile, 0, batchSize)
88
88
89
-
pfd, err := packages.GetPackageFileDescriptor(ctx, pf)
90
-
if err != nil {
89
+
if err := db.GetEngine(ctx).
90
+
Table("package_file").
91
+
Select("package_file.*").
92
+
Join("INNER", "package_version", "package_version.id = package_file.version_id").
93
+
Join("INNER", "package", "package.id = package_version.package_id").
94
+
Where(opts.toCond()).
95
+
Asc("package.lower_name", "package_version.created_unix").
96
+
Limit(batchSize, start).
97
+
Find(&beans); err != nil {
91
98
return err
92
99
}
100
+
if len(beans) == 0 {
101
+
return nil
102
+
}
103
+
start += len(beans)
93
104
94
-
iter(pfd)
105
+
for _, bean := range beans {
106
+
pfd, err := packages.GetPackageFileDescriptor(ctx, bean)
107
+
if err != nil {
108
+
return err
109
+
}
95
110
96
-
return nil
97
-
})
111
+
iter(pfd)
112
+
}
113
+
}
114
+
}
98
115
}
99
116
100
117
// GetDistributions gets all available distributions
+93
models/packages/debian/search_test.go
+93
models/packages/debian/search_test.go
···
1
+
// Copyright 2024 The Forgejo Authors. All rights reserved.
2
+
// SPDX-License-Identifier: GPL-3.0-or-later
3
+
4
+
package debian
5
+
6
+
import (
7
+
"strings"
8
+
"testing"
9
+
10
+
"code.gitea.io/gitea/models/db"
11
+
packages_model "code.gitea.io/gitea/models/packages"
12
+
"code.gitea.io/gitea/models/unittest"
13
+
user_model "code.gitea.io/gitea/models/user"
14
+
"code.gitea.io/gitea/modules/packages"
15
+
"code.gitea.io/gitea/modules/setting"
16
+
"code.gitea.io/gitea/modules/test"
17
+
packages_service "code.gitea.io/gitea/services/packages"
18
+
19
+
_ "code.gitea.io/gitea/models"
20
+
_ "code.gitea.io/gitea/models/actions"
21
+
_ "code.gitea.io/gitea/models/activities"
22
+
23
+
"github.com/stretchr/testify/assert"
24
+
"github.com/stretchr/testify/require"
25
+
)
26
+
27
+
func TestMain(m *testing.M) {
28
+
unittest.MainTest(m)
29
+
}
30
+
31
+
func preparePackage(t *testing.T, owner *user_model.User, name string) {
32
+
t.Helper()
33
+
34
+
data, err := packages.CreateHashedBufferFromReader(strings.NewReader("data"))
35
+
require.NoError(t, err)
36
+
37
+
_, _, err = packages_service.CreatePackageOrAddFileToExisting(
38
+
db.DefaultContext,
39
+
&packages_service.PackageCreationInfo{
40
+
PackageInfo: packages_service.PackageInfo{
41
+
Owner: owner,
42
+
PackageType: packages_model.TypeDebian,
43
+
Name: name,
44
+
},
45
+
Creator: owner,
46
+
},
47
+
&packages_service.PackageFileCreationInfo{
48
+
PackageFileInfo: packages_service.PackageFileInfo{
49
+
Filename: name,
50
+
},
51
+
Data: data,
52
+
Creator: owner,
53
+
IsLead: true,
54
+
},
55
+
)
56
+
57
+
require.NoError(t, err)
58
+
}
59
+
60
+
func TestSearchPackages(t *testing.T) {
61
+
require.NoError(t, unittest.PrepareTestDatabase())
62
+
defer test.MockVariableValue(&setting.Database.IterateBufferSize, 1)()
63
+
64
+
user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
65
+
user3 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3})
66
+
67
+
preparePackage(t, user2, "debian-1")
68
+
preparePackage(t, user2, "debian-2")
69
+
preparePackage(t, user3, "debian-1")
70
+
71
+
packageFiles := []string{}
72
+
require.NoError(t, SearchPackages(db.DefaultContext, &PackageSearchOptions{
73
+
OwnerID: user2.ID,
74
+
}, func(pfd *packages_model.PackageFileDescriptor) {
75
+
assert.NotNil(t, pfd)
76
+
packageFiles = append(packageFiles, pfd.File.Name)
77
+
}))
78
+
79
+
assert.Len(t, packageFiles, 2)
80
+
assert.Contains(t, packageFiles, "debian-1")
81
+
assert.Contains(t, packageFiles, "debian-2")
82
+
83
+
packageFiles = []string{}
84
+
require.NoError(t, SearchPackages(db.DefaultContext, &PackageSearchOptions{
85
+
OwnerID: user3.ID,
86
+
}, func(pfd *packages_model.PackageFileDescriptor) {
87
+
assert.NotNil(t, pfd)
88
+
packageFiles = append(packageFiles, pfd.File.Name)
89
+
}))
90
+
91
+
assert.Len(t, packageFiles, 1)
92
+
assert.Contains(t, packageFiles, "debian-1")
93
+
}