loading up the forgejo repo on tangled to test page performance
Merge pull request 'enh(issue search): sort by score and term based query for fuzzy search' (#5819) from snematoda/enh-issue-search into forgejo
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/5819
Reviewed-by: Otto <otto@codeberg.org>
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
+76
-31
+9
modules/indexer/internal/bleve/query.go
+9
modules/indexer/internal/bleve/query.go
···
19
19
return q
20
20
}
21
21
22
+
// MatchQuery generates a match query for the given phrase, field and analyzer
23
+
func MatchQuery(matchTerm, field, analyzer string, fuzziness int) *query.MatchQuery {
24
+
q := bleve.NewMatchQuery(matchTerm)
25
+
q.FieldVal = field
26
+
q.Analyzer = analyzer
27
+
q.Fuzziness = fuzziness
28
+
return q
29
+
}
30
+
22
31
// MatchPhraseQuery generates a match phrase query for the given phrase, field and analyzer
23
32
func MatchPhraseQuery(matchPhrase, field, analyzer string, fuzziness int) *query.MatchPhraseQuery {
24
33
q := bleve.NewMatchPhraseQuery(matchPhrase)
+18
-15
modules/indexer/issues/bleve/bleve.go
+18
-15
modules/indexer/issues/bleve/bleve.go
···
35
35
})
36
36
}
37
37
38
-
const (
39
-
maxBatchSize = 16
40
-
// fuzzyDenominator determines the levenshtein distance per each character of a keyword
41
-
fuzzyDenominator = 4
42
-
// see https://github.com/blevesearch/bleve/issues/1563#issuecomment-786822311
43
-
maxFuzziness = 2
44
-
)
38
+
const maxBatchSize = 16
45
39
46
40
// IndexerData an update to the issue indexer
47
41
type IndexerData internal.IndexerData
···
162
156
var queries []query.Query
163
157
164
158
if options.Keyword != "" {
165
-
fuzziness := 0
166
159
if options.IsFuzzyKeyword {
167
-
fuzziness = min(maxFuzziness, len(options.Keyword)/fuzzyDenominator)
160
+
fuzziness := 1
161
+
if kl := len(options.Keyword); kl > 3 {
162
+
fuzziness = 2
163
+
} else if kl < 2 {
164
+
fuzziness = 0
165
+
}
166
+
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
167
+
inner_bleve.MatchQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness),
168
+
inner_bleve.MatchQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness),
169
+
inner_bleve.MatchQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness),
170
+
}...))
171
+
} else {
172
+
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
173
+
inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, 0),
174
+
inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, 0),
175
+
inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, 0),
176
+
}...))
168
177
}
169
-
170
-
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
171
-
inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness),
172
-
inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness),
173
-
inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness),
174
-
}...))
175
178
}
176
179
177
180
if len(options.RepoIDs) > 0 || options.AllPublic {
+3
-1
modules/indexer/issues/dboptions.go
+3
-1
modules/indexer/issues/dboptions.go
+1
-1
modules/indexer/issues/elasticsearch/elasticsearch.go
+1
-1
modules/indexer/issues/elasticsearch/elasticsearch.go
+1
modules/indexer/issues/indexer.go
+1
modules/indexer/issues/indexer.go
+1
modules/indexer/issues/internal/model.go
+1
modules/indexer/issues/internal/model.go
+20
modules/indexer/issues/internal/tests/tests.go
+20
modules/indexer/issues/internal/tests/tests.go
···
126
126
},
127
127
SearchOptions: &internal.SearchOptions{
128
128
Keyword: "hello",
129
+
SortBy: internal.SortByCreatedDesc,
129
130
},
130
131
ExpectedIDs: []int64{1002, 1001, 1000},
131
132
ExpectedTotal: 3,
···
139
140
},
140
141
SearchOptions: &internal.SearchOptions{
141
142
Keyword: "hello world",
143
+
SortBy: internal.SortByCreatedDesc,
142
144
IsFuzzyKeyword: true,
143
145
},
144
146
ExpectedIDs: []int64{1002, 1001, 1000},
···
157
159
},
158
160
SearchOptions: &internal.SearchOptions{
159
161
Keyword: "hello",
162
+
SortBy: internal.SortByCreatedDesc,
160
163
RepoIDs: []int64{1, 4},
161
164
},
162
165
ExpectedIDs: []int64{1006, 1002, 1001},
···
175
178
},
176
179
SearchOptions: &internal.SearchOptions{
177
180
Keyword: "hello",
181
+
SortBy: internal.SortByCreatedDesc,
178
182
RepoIDs: []int64{1, 4},
179
183
AllPublic: true,
180
184
},
···
593
597
for i, v := range result.Hits {
594
598
if i < len(result.Hits)-1 {
595
599
assert.GreaterOrEqual(t, data[v.ID].DeadlineUnix, data[result.Hits[i+1].ID].DeadlineUnix)
600
+
}
601
+
}
602
+
},
603
+
},
604
+
{
605
+
Name: "SortByScore",
606
+
SearchOptions: &internal.SearchOptions{
607
+
Paginator: &db.ListOptionsAll,
608
+
SortBy: internal.SortByScore,
609
+
},
610
+
Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
611
+
assert.Equal(t, len(data), len(result.Hits))
612
+
assert.Equal(t, len(data), int(result.Total))
613
+
for i, v := range result.Hits {
614
+
if i < len(result.Hits)-1 {
615
+
assert.GreaterOrEqual(t, v.Score, result.Hits[i+1].Score)
596
616
}
597
617
}
598
618
},
+12
-6
modules/indexer/issues/meilisearch/meilisearch.go
+12
-6
modules/indexer/issues/meilisearch/meilisearch.go
···
208
208
query.And(inner_meilisearch.NewFilterLte("updated_unix", options.UpdatedBeforeUnix.Value()))
209
209
}
210
210
211
-
if options.SortBy == "" {
212
-
options.SortBy = internal.SortByCreatedAsc
213
-
}
214
-
sortBy := []string{
215
-
parseSortBy(options.SortBy),
216
-
"id:desc",
211
+
var sortBy []string
212
+
switch options.SortBy {
213
+
// sort by relevancy (no explicit sorting)
214
+
case internal.SortByScore:
215
+
fallthrough
216
+
case "":
217
+
sortBy = []string{}
218
+
default:
219
+
sortBy = []string{
220
+
parseSortBy(options.SortBy),
221
+
"id:desc",
222
+
}
217
223
}
218
224
219
225
skip, limit := indexer_internal.ParsePaginator(options.Paginator, maxTotalHits)
+4
modules/templates/util_string.go
+4
modules/templates/util_string.go
+1
options/locale/locale_en-US.ini
+1
options/locale/locale_en-US.ini
···
1586
1586
issues.filter_type.review_requested = Review requested
1587
1587
issues.filter_type.reviewed_by_you = Reviewed by you
1588
1588
issues.filter_sort = Sort
1589
+
issues.filter_sort.relevance = Relevance
1589
1590
issues.filter_sort.latest = Newest
1590
1591
issues.filter_sort.oldest = Oldest
1591
1592
issues.filter_sort.recentupdate = Recently updated
+6
-8
templates/repo/issue/filter_list.tmpl
+6
-8
templates/repo/issue/filter_list.tmpl
···
146
146
</span>
147
147
{{svg "octicon-triangle-down" 14 "dropdown icon"}}
148
148
<div class="menu">
149
-
<a rel="nofollow" class="{{if or (eq .SortType "latest") (not .SortType)}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=latest&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.latest"}}</a>
150
-
<a rel="nofollow" class="{{if eq .SortType "oldest"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=oldest&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.oldest"}}</a>
151
-
<a rel="nofollow" class="{{if eq .SortType "recentupdate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=recentupdate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.recentupdate"}}</a>
152
-
<a rel="nofollow" class="{{if eq .SortType "leastupdate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=leastupdate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.leastupdate"}}</a>
153
-
<a rel="nofollow" class="{{if eq .SortType "mostcomment"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=mostcomment&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.mostcomment"}}</a>
154
-
<a rel="nofollow" class="{{if eq .SortType "leastcomment"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=leastcomment&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.leastcomment"}}</a>
155
-
<a rel="nofollow" class="{{if eq .SortType "nearduedate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=nearduedate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.nearduedate"}}</a>
156
-
<a rel="nofollow" class="{{if eq .SortType "farduedate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=farduedate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.farduedate"}}</a>
149
+
<a rel="nofollow" class="{{if or (eq .SortType "relevance") (not .SortType)}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=relevency&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.relevance"}}</a>
150
+
{{$o := .}}
151
+
{{range $opt := StringUtils.Make "latest" "oldest" "recentupdate" "mostcomment" "leastcomment" "nearduedate" "farduedate"}}
152
+
{{$text := ctx.Locale.Tr (printf "repo.issues.filter_sort.%s" $opt)}}
153
+
<a rel="nofollow" class="{{if eq $o.SortType $opt}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort={{$opt}}&state={{$.State}}&labels={{$o.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{$text}}</a>
154
+
{{end}}
157
155
</div>
158
156
</div>