loading up the forgejo repo on tangled to test page performance
Merge pull request '[PORT] Refactor code_indexer to use an SearchOptions struct for PerformSearch (gitea#29724)' (#2686) from snematoda/port-29724 into forgejo
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/2686
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
+87
-40
+1
-1
models/issues/issue_update.go
+1
-1
models/issues/issue_update.go
+13
-13
modules/indexer/code/bleve/bleve.go
+13
-13
modules/indexer/code/bleve/bleve.go
···
142
142
return err
143
143
}
144
144
if size, err = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64); err != nil {
145
-
return fmt.Errorf("Misformatted git cat-file output: %w", err)
145
+
return fmt.Errorf("misformatted git cat-file output: %w", err)
146
146
}
147
147
}
148
148
···
233
233
234
234
// Search searches for files in the specified repo.
235
235
// Returns the matching file-paths
236
-
func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
236
+
func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
237
237
var (
238
238
indexerQuery query.Query
239
239
keywordQuery query.Query
240
240
)
241
241
242
-
if isFuzzy {
243
-
phraseQuery := bleve.NewMatchPhraseQuery(keyword)
242
+
if opts.IsKeywordFuzzy {
243
+
phraseQuery := bleve.NewMatchPhraseQuery(opts.Keyword)
244
244
phraseQuery.FieldVal = "Content"
245
245
phraseQuery.Analyzer = repoIndexerAnalyzer
246
246
keywordQuery = phraseQuery
247
247
} else {
248
-
prefixQuery := bleve.NewPrefixQuery(keyword)
248
+
prefixQuery := bleve.NewPrefixQuery(opts.Keyword)
249
249
prefixQuery.FieldVal = "Content"
250
250
keywordQuery = prefixQuery
251
251
}
252
252
253
-
if len(repoIDs) > 0 {
254
-
repoQueries := make([]query.Query, 0, len(repoIDs))
255
-
for _, repoID := range repoIDs {
253
+
if len(opts.RepoIDs) > 0 {
254
+
repoQueries := make([]query.Query, 0, len(opts.RepoIDs))
255
+
for _, repoID := range opts.RepoIDs {
256
256
repoQueries = append(repoQueries, inner_bleve.NumericEqualityQuery(repoID, "RepoID"))
257
257
}
258
258
···
266
266
267
267
// Save for reuse without language filter
268
268
facetQuery := indexerQuery
269
-
if len(language) > 0 {
270
-
languageQuery := bleve.NewMatchQuery(language)
269
+
if len(opts.Language) > 0 {
270
+
languageQuery := bleve.NewMatchQuery(opts.Language)
271
271
languageQuery.FieldVal = "Language"
272
272
languageQuery.Analyzer = analyzer_keyword.Name
273
273
···
277
277
)
278
278
}
279
279
280
-
from := (page - 1) * pageSize
280
+
from, pageSize := opts.GetSkipTake()
281
281
searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
282
282
searchRequest.Fields = []string{"Content", "RepoID", "Language", "CommitID", "UpdatedAt"}
283
283
searchRequest.IncludeLocations = true
284
284
285
-
if len(language) == 0 {
285
+
if len(opts.Language) == 0 {
286
286
searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
287
287
}
288
288
···
326
326
}
327
327
328
328
searchResultLanguages := make([]*internal.SearchResultLanguages, 0, 10)
329
-
if len(language) > 0 {
329
+
if len(opts.Language) > 0 {
330
330
// Use separate query to go get all language counts
331
331
facetRequest := bleve.NewSearchRequestOptions(facetQuery, 1, 0, false)
332
332
facetRequest.Fields = []string{"Content", "RepoID", "Language", "CommitID", "UpdatedAt"}
+11
-15
modules/indexer/code/elasticsearch/elasticsearch.go
+11
-15
modules/indexer/code/elasticsearch/elasticsearch.go
···
281
281
}
282
282
283
283
// Search searches for codes and language stats by given conditions.
284
-
func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
284
+
func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
285
285
searchType := esMultiMatchTypePhrasePrefix
286
-
if isFuzzy {
286
+
if opts.IsKeywordFuzzy {
287
287
searchType = esMultiMatchTypeBestFields
288
288
}
289
289
290
-
kwQuery := elastic.NewMultiMatchQuery(keyword, "content").Type(searchType)
290
+
kwQuery := elastic.NewMultiMatchQuery(opts.Keyword, "content").Type(searchType)
291
291
query := elastic.NewBoolQuery()
292
292
query = query.Must(kwQuery)
293
-
if len(repoIDs) > 0 {
294
-
repoStrs := make([]any, 0, len(repoIDs))
295
-
for _, repoID := range repoIDs {
293
+
if len(opts.RepoIDs) > 0 {
294
+
repoStrs := make([]any, 0, len(opts.RepoIDs))
295
+
for _, repoID := range opts.RepoIDs {
296
296
repoStrs = append(repoStrs, repoID)
297
297
}
298
298
repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...)
···
300
300
}
301
301
302
302
var (
303
-
start int
304
-
kw = "<em>" + keyword + "</em>"
305
-
aggregation = elastic.NewTermsAggregation().Field("language").Size(10).OrderByCountDesc()
303
+
start, pageSize = opts.GetSkipTake()
304
+
kw = "<em>" + opts.Keyword + "</em>"
305
+
aggregation = elastic.NewTermsAggregation().Field("language").Size(10).OrderByCountDesc()
306
306
)
307
307
308
-
if page > 0 {
309
-
start = (page - 1) * pageSize
310
-
}
311
-
312
-
if len(language) == 0 {
308
+
if len(opts.Language) == 0 {
313
309
searchResult, err := b.inner.Client.Search().
314
310
Index(b.inner.VersionedIndexName()).
315
311
Aggregation("language", aggregation).
···
330
326
return convertResult(searchResult, kw, pageSize)
331
327
}
332
328
333
-
langQuery := elastic.NewMatchQuery("language", language)
329
+
langQuery := elastic.NewMatchQuery("language", opts.Language)
334
330
countResult, err := b.inner.Client.Search().
335
331
Index(b.inner.VersionedIndexName()).
336
332
Aggregation("language", aggregation).
+1
-1
modules/indexer/code/git.go
+1
-1
modules/indexer/code/git.go
···
32
32
33
33
needGenesis := len(status.CommitSha) == 0
34
34
if !needGenesis {
35
-
hasAncestorCmd := git.NewCommand(ctx, "merge-base").AddDynamicArguments(repo.CodeIndexerStatus.CommitSha, revision)
35
+
hasAncestorCmd := git.NewCommand(ctx, "merge-base").AddDynamicArguments(status.CommitSha, revision)
36
36
stdout, _, _ := hasAncestorCmd.RunStdString(&git.RunOpts{Dir: repo.RepoPath()})
37
37
needGenesis = len(stdout) == 0
38
38
}
+10
-1
modules/indexer/code/indexer_test.go
+10
-1
modules/indexer/code/indexer_test.go
···
8
8
"os"
9
9
"testing"
10
10
11
+
"code.gitea.io/gitea/models/db"
11
12
"code.gitea.io/gitea/models/unittest"
12
13
"code.gitea.io/gitea/modules/git"
13
14
"code.gitea.io/gitea/modules/indexer/code/bleve"
···
70
71
71
72
for _, kw := range keywords {
72
73
t.Run(kw.Keyword, func(t *testing.T) {
73
-
total, res, langs, err := indexer.Search(context.TODO(), kw.RepoIDs, "", kw.Keyword, 1, 10, true)
74
+
total, res, langs, err := indexer.Search(context.TODO(), &internal.SearchOptions{
75
+
RepoIDs: kw.RepoIDs,
76
+
Keyword: kw.Keyword,
77
+
Paginator: &db.ListOptions{
78
+
Page: 1,
79
+
PageSize: 10,
80
+
},
81
+
IsKeywordFuzzy: true,
82
+
})
74
83
assert.NoError(t, err)
75
84
assert.Len(t, kw.IDs, int(total))
76
85
assert.Len(t, langs, kw.Langs)
+13
-2
modules/indexer/code/internal/indexer.go
+13
-2
modules/indexer/code/internal/indexer.go
···
7
7
"context"
8
8
"fmt"
9
9
10
+
"code.gitea.io/gitea/models/db"
10
11
repo_model "code.gitea.io/gitea/models/repo"
11
12
"code.gitea.io/gitea/modules/indexer/internal"
12
13
)
···
16
17
internal.Indexer
17
18
Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error
18
19
Delete(ctx context.Context, repoID int64) error
19
-
Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
20
+
Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error)
21
+
}
22
+
23
+
type SearchOptions struct {
24
+
RepoIDs []int64
25
+
Keyword string
26
+
Language string
27
+
28
+
IsKeywordFuzzy bool
29
+
30
+
db.Paginator
20
31
}
21
32
22
33
// NewDummyIndexer returns a dummy indexer
···
38
49
return fmt.Errorf("indexer is not ready")
39
50
}
40
51
41
-
func (d *dummyIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
52
+
func (d *dummyIndexer) Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error) {
42
53
return 0, nil, nil, fmt.Errorf("indexer is not ready")
43
54
}
+5
-3
modules/indexer/code/search.go
+5
-3
modules/indexer/code/search.go
···
32
32
33
33
type SearchResultLanguages = internal.SearchResultLanguages
34
34
35
+
type SearchOptions = internal.SearchOptions
36
+
35
37
func indices(content string, selectionStartIndex, selectionEndIndex int) (int, int) {
36
38
startIndex := selectionStartIndex
37
39
numLinesBefore := 0
···
125
127
126
128
// PerformSearch perform a search on a repository
127
129
// if isFuzzy is true set the Damerau-Levenshtein distance from 0 to 2
128
-
func PerformSearch(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int, []*Result, []*internal.SearchResultLanguages, error) {
129
-
if len(keyword) == 0 {
130
+
func PerformSearch(ctx context.Context, opts *SearchOptions) (int, []*Result, []*SearchResultLanguages, error) {
131
+
if opts == nil || len(opts.Keyword) == 0 {
130
132
return 0, nil, nil, nil
131
133
}
132
134
133
-
total, results, resultLanguages, err := (*globalIndexer.Load()).Search(ctx, repoIDs, language, keyword, page, pageSize, isFuzzy)
135
+
total, results, resultLanguages, err := (*globalIndexer.Load()).Search(ctx, opts)
134
136
if err != nil {
135
137
return 0, nil, nil, err
136
138
}
+11
-1
routers/web/explore/code.go
+11
-1
routers/web/explore/code.go
···
6
6
import (
7
7
"net/http"
8
8
9
+
"code.gitea.io/gitea/models/db"
9
10
repo_model "code.gitea.io/gitea/models/repo"
10
11
"code.gitea.io/gitea/modules/base"
11
12
code_indexer "code.gitea.io/gitea/modules/indexer/code"
···
77
78
)
78
79
79
80
if (len(repoIDs) > 0) || isAdmin {
80
-
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isFuzzy)
81
+
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{
82
+
RepoIDs: repoIDs,
83
+
Keyword: keyword,
84
+
IsKeywordFuzzy: isFuzzy,
85
+
Language: language,
86
+
Paginator: &db.ListOptions{
87
+
Page: page,
88
+
PageSize: setting.UI.RepoSearchPagingNum,
89
+
},
90
+
})
81
91
if err != nil {
82
92
if code_indexer.IsAvailable(ctx) {
83
93
ctx.ServerError("SearchResults", err)
+11
-2
routers/web/repo/search.go
+11
-2
routers/web/repo/search.go
···
6
6
import (
7
7
"net/http"
8
8
9
+
"code.gitea.io/gitea/models/db"
9
10
"code.gitea.io/gitea/modules/base"
10
11
code_indexer "code.gitea.io/gitea/modules/indexer/code"
11
12
"code.gitea.io/gitea/modules/setting"
···
43
44
if setting.Indexer.RepoIndexerEnabled {
44
45
ctx.Data["CodeIndexerEnabled"] = true
45
46
46
-
total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch(ctx, []int64{ctx.Repo.Repository.ID},
47
-
language, keyword, page, setting.UI.RepoSearchPagingNum, isFuzzy)
47
+
total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{
48
+
RepoIDs: []int64{ctx.Repo.Repository.ID},
49
+
Keyword: keyword,
50
+
IsKeywordFuzzy: isFuzzy,
51
+
Language: language,
52
+
Paginator: &db.ListOptions{
53
+
Page: page,
54
+
PageSize: setting.UI.RepoSearchPagingNum,
55
+
},
56
+
})
48
57
if err != nil {
49
58
if code_indexer.IsAvailable(ctx) {
50
59
ctx.ServerError("SearchResults", err)
+11
-1
routers/web/user/code.go
+11
-1
routers/web/user/code.go
···
6
6
import (
7
7
"net/http"
8
8
9
+
"code.gitea.io/gitea/models/db"
9
10
repo_model "code.gitea.io/gitea/models/repo"
10
11
"code.gitea.io/gitea/modules/base"
11
12
code_indexer "code.gitea.io/gitea/modules/indexer/code"
···
75
76
)
76
77
77
78
if len(repoIDs) > 0 {
78
-
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isFuzzy)
79
+
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{
80
+
RepoIDs: repoIDs,
81
+
Keyword: keyword,
82
+
IsKeywordFuzzy: isFuzzy,
83
+
Language: language,
84
+
Paginator: &db.ListOptions{
85
+
Page: page,
86
+
PageSize: setting.UI.RepoSearchPagingNum,
87
+
},
88
+
})
79
89
if err != nil {
80
90
if code_indexer.IsAvailable(ctx) {
81
91
ctx.ServerError("SearchResults", err)