loading up the forgejo repo on tangled to test page performance
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge pull request 'fix: use buffered iterate for debian searchpackages' (#5702) from gusted/forgejo-buffered-iterate into forgejo

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/5702
Reviewed-by: Otto <otto@codeberg.org>

Otto 4a41c71d 2bdd7cb0

+124 -14
+31 -14
models/packages/debian/search.go
··· 10 10 "code.gitea.io/gitea/models/db" 11 11 "code.gitea.io/gitea/models/packages" 12 12 debian_module "code.gitea.io/gitea/modules/packages/debian" 13 + "code.gitea.io/gitea/modules/setting" 13 14 14 15 "xorm.io/builder" 15 16 ) ··· 76 77 77 78 // SearchPackages gets the packages matching the search options 78 79 func SearchPackages(ctx context.Context, opts *PackageSearchOptions, iter func(*packages.PackageFileDescriptor)) error { 79 - return db.GetEngine(ctx). 80 - Table("package_file"). 81 - Select("package_file.*"). 82 - Join("INNER", "package_version", "package_version.id = package_file.version_id"). 83 - Join("INNER", "package", "package.id = package_version.package_id"). 84 - Where(opts.toCond()). 85 - Asc("package.lower_name", "package_version.created_unix"). 86 - Iterate(new(packages.PackageFile), func(_ int, bean any) error { 87 - pf := bean.(*packages.PackageFile) 80 + var start int 81 + batchSize := setting.Database.IterateBufferSize 82 + for { 83 + select { 84 + case <-ctx.Done(): 85 + return ctx.Err() 86 + default: 87 + beans := make([]*packages.PackageFile, 0, batchSize) 88 88 89 - pfd, err := packages.GetPackageFileDescriptor(ctx, pf) 90 - if err != nil { 89 + if err := db.GetEngine(ctx). 90 + Table("package_file"). 91 + Select("package_file.*"). 92 + Join("INNER", "package_version", "package_version.id = package_file.version_id"). 93 + Join("INNER", "package", "package.id = package_version.package_id"). 94 + Where(opts.toCond()). 95 + Asc("package.lower_name", "package_version.created_unix"). 96 + Limit(batchSize, start). 97 + Find(&beans); err != nil { 91 98 return err 92 99 } 100 + if len(beans) == 0 { 101 + return nil 102 + } 103 + start += len(beans) 93 104 94 - iter(pfd) 105 + for _, bean := range beans { 106 + pfd, err := packages.GetPackageFileDescriptor(ctx, bean) 107 + if err != nil { 108 + return err 109 + } 95 110 96 - return nil 97 - }) 111 + iter(pfd) 112 + } 113 + } 114 + } 98 115 } 99 116 100 117 // GetDistributions gets all available distributions
+93
models/packages/debian/search_test.go
··· 1 + // Copyright 2024 The Forgejo Authors. All rights reserved. 2 + // SPDX-License-Identifier: GPL-3.0-or-later 3 + 4 + package debian 5 + 6 + import ( 7 + "strings" 8 + "testing" 9 + 10 + "code.gitea.io/gitea/models/db" 11 + packages_model "code.gitea.io/gitea/models/packages" 12 + "code.gitea.io/gitea/models/unittest" 13 + user_model "code.gitea.io/gitea/models/user" 14 + "code.gitea.io/gitea/modules/packages" 15 + "code.gitea.io/gitea/modules/setting" 16 + "code.gitea.io/gitea/modules/test" 17 + packages_service "code.gitea.io/gitea/services/packages" 18 + 19 + _ "code.gitea.io/gitea/models" 20 + _ "code.gitea.io/gitea/models/actions" 21 + _ "code.gitea.io/gitea/models/activities" 22 + 23 + "github.com/stretchr/testify/assert" 24 + "github.com/stretchr/testify/require" 25 + ) 26 + 27 + func TestMain(m *testing.M) { 28 + unittest.MainTest(m) 29 + } 30 + 31 + func preparePackage(t *testing.T, owner *user_model.User, name string) { 32 + t.Helper() 33 + 34 + data, err := packages.CreateHashedBufferFromReader(strings.NewReader("data")) 35 + require.NoError(t, err) 36 + 37 + _, _, err = packages_service.CreatePackageOrAddFileToExisting( 38 + db.DefaultContext, 39 + &packages_service.PackageCreationInfo{ 40 + PackageInfo: packages_service.PackageInfo{ 41 + Owner: owner, 42 + PackageType: packages_model.TypeDebian, 43 + Name: name, 44 + }, 45 + Creator: owner, 46 + }, 47 + &packages_service.PackageFileCreationInfo{ 48 + PackageFileInfo: packages_service.PackageFileInfo{ 49 + Filename: name, 50 + }, 51 + Data: data, 52 + Creator: owner, 53 + IsLead: true, 54 + }, 55 + ) 56 + 57 + require.NoError(t, err) 58 + } 59 + 60 + func TestSearchPackages(t *testing.T) { 61 + require.NoError(t, unittest.PrepareTestDatabase()) 62 + defer test.MockVariableValue(&setting.Database.IterateBufferSize, 1)() 63 + 64 + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) 65 + user3 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}) 66 + 67 + preparePackage(t, user2, "debian-1") 68 + preparePackage(t, user2, "debian-2") 69 + preparePackage(t, user3, "debian-1") 70 + 71 + packageFiles := []string{} 72 + require.NoError(t, SearchPackages(db.DefaultContext, &PackageSearchOptions{ 73 + OwnerID: user2.ID, 74 + }, func(pfd *packages_model.PackageFileDescriptor) { 75 + assert.NotNil(t, pfd) 76 + packageFiles = append(packageFiles, pfd.File.Name) 77 + })) 78 + 79 + assert.Len(t, packageFiles, 2) 80 + assert.Contains(t, packageFiles, "debian-1") 81 + assert.Contains(t, packageFiles, "debian-2") 82 + 83 + packageFiles = []string{} 84 + require.NoError(t, SearchPackages(db.DefaultContext, &PackageSearchOptions{ 85 + OwnerID: user3.ID, 86 + }, func(pfd *packages_model.PackageFileDescriptor) { 87 + assert.NotNil(t, pfd) 88 + packageFiles = append(packageFiles, pfd.File.Name) 89 + })) 90 + 91 + assert.Len(t, packageFiles, 1) 92 + assert.Contains(t, packageFiles, "debian-1") 93 + }