loading up the forgejo repo on tangled to test page performance
Merge pull request 'feat: support regexp in git-grep search' (#4968) from yoctozepto/git-grep-regexp into forgejo
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/4968
Reviewed-by: Shiny Nematoda <snematoda@noreply.codeberg.org>
+182
-56
+21
-4
modules/git/grep.go
+21
-4
modules/git/grep.go
···
27
27
HighlightedRanges [][3]int
28
28
}
29
29
30
+
type grepMode int
31
+
32
+
const (
33
+
FixedGrepMode grepMode = iota
34
+
FixedAnyGrepMode
35
+
RegExpGrepMode
36
+
)
37
+
30
38
type GrepOptions struct {
31
39
RefName string
32
40
MaxResultLimit int
33
41
MatchesPerFile int
34
42
ContextLineNumber int
35
-
IsFuzzy bool
43
+
Mode grepMode
36
44
PathSpec []setting.Glob
37
45
}
38
46
···
74
82
var results []*GrepResult
75
83
// -I skips binary files
76
84
cmd := NewCommand(ctx, "grep",
77
-
"-I", "--null", "--break", "--heading", "--column",
78
-
"--fixed-strings", "--line-number", "--ignore-case", "--full-name")
85
+
"-I", "--null", "--break", "--heading",
86
+
"--line-number", "--ignore-case", "--full-name")
87
+
if opts.Mode == RegExpGrepMode {
88
+
// No `--column` -- regexp mode does not support highlighting in the
89
+
// current implementation as the length of the match is unknown from
90
+
// `grep` but required for highlighting.
91
+
cmd.AddArguments("--perl-regexp")
92
+
} else {
93
+
cmd.AddArguments("--fixed-strings", "--column")
94
+
}
79
95
cmd.AddOptionValues("--context", fmt.Sprint(opts.ContextLineNumber))
80
96
cmd.AddOptionValues("--max-count", fmt.Sprint(opts.MatchesPerFile))
81
97
words := []string{search}
82
-
if opts.IsFuzzy {
98
+
if opts.Mode == FixedAnyGrepMode {
83
99
words = strings.Fields(search)
84
100
}
85
101
for _, word := range words {
···
148
164
if lineNum, lineCode, ok := strings.Cut(line, "\x00"); ok {
149
165
lineNumInt, _ := strconv.Atoi(lineNum)
150
166
res.LineNumbers = append(res.LineNumbers, lineNumInt)
167
+
// We support highlighting only when `--column` parameter is used.
151
168
if lineCol, lineCode2, ok := strings.Cut(lineCode, "\x00"); ok {
152
169
lineColInt, _ := strconv.Atoi(lineCol)
153
170
start := lineColInt - 1
+31
modules/git/grep_test.go
+31
modules/git/grep_test.go
···
201
201
assert.Len(t, res, 1)
202
202
assert.Equal(t, "A", res[0].LineCodes[0])
203
203
}
204
+
205
+
func TestGrepCanHazRegexOnDemand(t *testing.T) {
206
+
tmpDir := t.TempDir()
207
+
208
+
err := InitRepository(DefaultContext, tmpDir, false, Sha1ObjectFormat.Name())
209
+
require.NoError(t, err)
210
+
211
+
gitRepo, err := openRepositoryWithDefaultContext(tmpDir)
212
+
require.NoError(t, err)
213
+
defer gitRepo.Close()
214
+
215
+
require.NoError(t, os.WriteFile(path.Join(tmpDir, "matching"), []byte("It's a match!"), 0o666))
216
+
require.NoError(t, os.WriteFile(path.Join(tmpDir, "not-matching"), []byte("Orisitamatch?"), 0o666))
217
+
218
+
err = AddChanges(tmpDir, true)
219
+
require.NoError(t, err)
220
+
221
+
err = CommitChanges(tmpDir, CommitChangesOptions{Message: "Add fixtures for regexp test"})
222
+
require.NoError(t, err)
223
+
224
+
// should find nothing by default...
225
+
res, err := GrepSearch(context.Background(), gitRepo, "\\bmatch\\b", GrepOptions{})
226
+
require.NoError(t, err)
227
+
assert.Empty(t, res)
228
+
229
+
// ... unless configured explicitly
230
+
res, err = GrepSearch(context.Background(), gitRepo, "\\bmatch\\b", GrepOptions{Mode: RegExpGrepMode})
231
+
require.NoError(t, err)
232
+
assert.Len(t, res, 1)
233
+
assert.Equal(t, "matching", res[0].Filename)
234
+
}
+2
options/locale/locale_en-US.ini
+2
options/locale/locale_en-US.ini
···
173
173
union_tooltip = Include results that match any of the whitespace seperated keywords
174
174
exact = Exact
175
175
exact_tooltip = Include only results that match the exact search term
176
+
regexp = RegExp
177
+
regexp_tooltip = Interpret the search term as a regular expression
176
178
repo_kind = Search repos...
177
179
user_kind = Search users...
178
180
org_kind = Search orgs...
+9
-1
routers/web/explore/code.go
+9
-1
routers/web/explore/code.go
···
36
36
keyword := ctx.FormTrim("q")
37
37
38
38
isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true)
39
+
if mode := ctx.FormTrim("mode"); len(mode) > 0 {
40
+
isFuzzy = mode == "fuzzy"
41
+
}
39
42
40
43
ctx.Data["Keyword"] = keyword
41
44
ctx.Data["Language"] = language
42
-
ctx.Data["IsFuzzy"] = isFuzzy
45
+
ctx.Data["CodeSearchOptions"] = []string{"exact", "fuzzy"}
46
+
if isFuzzy {
47
+
ctx.Data["CodeSearchMode"] = "fuzzy"
48
+
} else {
49
+
ctx.Data["CodeSearchMode"] = "exact"
50
+
}
43
51
ctx.Data["PageIsViewCode"] = true
44
52
45
53
if keyword == "" {
+52
-6
routers/web/repo/search.go
+52
-6
routers/web/repo/search.go
···
17
17
18
18
const tplSearch base.TplName = "repo/search"
19
19
20
+
type searchMode int
21
+
22
+
const (
23
+
ExactSearchMode searchMode = iota
24
+
FuzzySearchMode
25
+
RegExpSearchMode
26
+
)
27
+
28
+
func searchModeFromString(s string) searchMode {
29
+
switch s {
30
+
case "fuzzy", "union":
31
+
return FuzzySearchMode
32
+
case "regexp":
33
+
return RegExpSearchMode
34
+
default:
35
+
return ExactSearchMode
36
+
}
37
+
}
38
+
39
+
func (m searchMode) String() string {
40
+
switch m {
41
+
case ExactSearchMode:
42
+
return "exact"
43
+
case FuzzySearchMode:
44
+
return "fuzzy"
45
+
case RegExpSearchMode:
46
+
return "regexp"
47
+
default:
48
+
panic("cannot happen")
49
+
}
50
+
}
51
+
20
52
// Search render repository search page
21
53
func Search(ctx *context.Context) {
22
54
language := ctx.FormTrim("l")
23
55
keyword := ctx.FormTrim("q")
24
56
25
-
isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true)
57
+
mode := ExactSearchMode
58
+
if modeStr := ctx.FormString("mode"); len(modeStr) > 0 {
59
+
mode = searchModeFromString(modeStr)
60
+
} else if ctx.FormOptionalBool("fuzzy").ValueOrDefault(true) { // for backward compatibility in links
61
+
mode = FuzzySearchMode
62
+
}
26
63
27
64
ctx.Data["Keyword"] = keyword
28
65
ctx.Data["Language"] = language
29
-
ctx.Data["IsFuzzy"] = isFuzzy
66
+
ctx.Data["CodeSearchMode"] = mode.String()
30
67
ctx.Data["PageIsViewCode"] = true
31
68
32
69
if keyword == "" {
···
47
84
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, &code_indexer.SearchOptions{
48
85
RepoIDs: []int64{ctx.Repo.Repository.ID},
49
86
Keyword: keyword,
50
-
IsKeywordFuzzy: isFuzzy,
87
+
IsKeywordFuzzy: mode == FuzzySearchMode,
51
88
Language: language,
52
89
Paginator: &db.ListOptions{
53
90
Page: page,
···
63
100
} else {
64
101
ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable(ctx)
65
102
}
103
+
ctx.Data["CodeSearchOptions"] = []string{"exact", "fuzzy"}
66
104
} else {
67
-
res, err := git.GrepSearch(ctx, ctx.Repo.GitRepo, keyword, git.GrepOptions{
105
+
grepOpt := git.GrepOptions{
68
106
ContextLineNumber: 1,
69
-
IsFuzzy: isFuzzy,
70
107
RefName: ctx.Repo.RefName,
71
-
})
108
+
}
109
+
switch mode {
110
+
case FuzzySearchMode:
111
+
grepOpt.Mode = git.FixedAnyGrepMode
112
+
ctx.Data["CodeSearchMode"] = "union"
113
+
case RegExpSearchMode:
114
+
grepOpt.Mode = git.RegExpGrepMode
115
+
}
116
+
res, err := git.GrepSearch(ctx, ctx.Repo.GitRepo, keyword, grepOpt)
72
117
if err != nil {
73
118
ctx.ServerError("GrepSearch", err)
74
119
return
···
88
133
Lines: code_indexer.HighlightSearchResultCode(r.Filename, r.LineNumbers, r.HighlightedRanges, strings.Join(r.LineCodes, "\n")),
89
134
})
90
135
}
136
+
ctx.Data["CodeSearchOptions"] = []string{"exact", "union", "regexp"}
91
137
}
92
138
93
139
ctx.Data["CodeIndexerDisabled"] = !setting.Indexer.RepoIndexerEnabled
+9
-1
routers/web/user/code.go
+9
-1
routers/web/user/code.go
···
41
41
keyword := ctx.FormTrim("q")
42
42
43
43
isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true)
44
+
if mode := ctx.FormTrim("mode"); len(mode) > 0 {
45
+
isFuzzy = mode == "fuzzy"
46
+
}
44
47
45
48
ctx.Data["Keyword"] = keyword
46
49
ctx.Data["Language"] = language
47
-
ctx.Data["IsFuzzy"] = isFuzzy
50
+
ctx.Data["CodeSearchOptions"] = []string{"exact", "fuzzy"}
51
+
if isFuzzy {
52
+
ctx.Data["CodeSearchMode"] = "fuzzy"
53
+
} else {
54
+
ctx.Data["CodeSearchMode"] = "exact"
55
+
}
48
56
ctx.Data["IsCodePage"] = true
49
57
50
58
if keyword == "" {
+1
-1
services/wiki/wiki.go
+1
-1
services/wiki/wiki.go
+1
-1
templates/repo/search.tmpl
+1
-1
templates/repo/search.tmpl
···
5
5
{{if $.CodeIndexerDisabled}}
6
6
{{$branchURLPrefix := printf "%s/search/branch/" $.RepoLink}}
7
7
{{$tagURLPrefix := printf "%s/search/tag/" $.RepoLink}}
8
-
{{$suffix := printf "?q=%s&fuzzy=%t" (.Keyword|QueryEscape) .IsFuzzy}}
8
+
{{$suffix := printf "?q=%s&mode=%s" (.Keyword|QueryEscape) .CodeSearchMode}}
9
9
{{template "repo/branch_dropdown" dict "root" . "ContainerClasses" "tw-mb-3" "branchURLPrefix" $branchURLPrefix "branchURLSuffix" $suffix "tagURLPrefix" $tagURLPrefix "tagURLSuffix" $suffix}}
10
10
{{end}}
11
11
{{template "shared/search/code/search" .}}
+22
-24
tests/integration/repo_search_test.go
+22
-24
tests/integration/repo_search_test.go
···
79
79
}
80
80
81
81
testSearch(t, "/user2/glob/search?q=loren&page=1", []string{"a.txt"}, indexer)
82
-
testSearch(t, "/user2/glob/search?q=loren&page=1&fuzzy=false", []string{"a.txt"}, indexer)
82
+
testSearch(t, "/user2/glob/search?q=loren&page=1&mode=exact", []string{"a.txt"}, indexer)
83
83
84
84
if indexer {
85
85
// fuzzy search: matches both file3 (x/b.txt) and file1 (a.txt)
86
86
// when indexer is enabled
87
-
testSearch(t, "/user2/glob/search?q=file3&page=1", []string{"x/b.txt", "a.txt"}, indexer)
88
-
testSearch(t, "/user2/glob/search?q=file4&page=1", []string{"x/b.txt", "a.txt"}, indexer)
89
-
testSearch(t, "/user2/glob/search?q=file5&page=1", []string{"x/b.txt", "a.txt"}, indexer)
87
+
testSearch(t, "/user2/glob/search?q=file3&mode=fuzzy&page=1", []string{"x/b.txt", "a.txt"}, indexer)
88
+
testSearch(t, "/user2/glob/search?q=file4&mode=fuzzy&page=1", []string{"x/b.txt", "a.txt"}, indexer)
89
+
testSearch(t, "/user2/glob/search?q=file5&mode=fuzzy&page=1", []string{"x/b.txt", "a.txt"}, indexer)
90
90
} else {
91
91
// fuzzy search: Union/OR of all the keywords
92
92
// when indexer is disabled
93
-
testSearch(t, "/user2/glob/search?q=file3+file1&page=1", []string{"a.txt", "x/b.txt"}, indexer)
94
-
testSearch(t, "/user2/glob/search?q=file4&page=1", []string{}, indexer)
95
-
testSearch(t, "/user2/glob/search?q=file5&page=1", []string{}, indexer)
93
+
testSearch(t, "/user2/glob/search?q=file3+file1&mode=union&page=1", []string{"a.txt", "x/b.txt"}, indexer)
94
+
testSearch(t, "/user2/glob/search?q=file4&mode=union&page=1", []string{}, indexer)
95
+
testSearch(t, "/user2/glob/search?q=file5&mode=union&page=1", []string{}, indexer)
96
96
}
97
97
98
-
testSearch(t, "/user2/glob/search?q=file3&page=1&fuzzy=false", []string{"x/b.txt"}, indexer)
99
-
testSearch(t, "/user2/glob/search?q=file4&page=1&fuzzy=false", []string{}, indexer)
100
-
testSearch(t, "/user2/glob/search?q=file5&page=1&fuzzy=false", []string{}, indexer)
98
+
testSearch(t, "/user2/glob/search?q=file3&page=1&mode=exact", []string{"x/b.txt"}, indexer)
99
+
testSearch(t, "/user2/glob/search?q=file4&page=1&mode=exact", []string{}, indexer)
100
+
testSearch(t, "/user2/glob/search?q=file5&page=1&mode=exact", []string{}, indexer)
101
101
}
102
102
103
103
func testSearch(t *testing.T, url string, expected []string, indexer bool) {
···
113
113
branchDropdown := container.Find(".js-branch-tag-selector")
114
114
assert.EqualValues(t, indexer, len(branchDropdown.Nodes) == 0)
115
115
116
-
// if indexer is disabled "fuzzy" should be displayed as "union"
117
-
expectedFuzzy := "Fuzzy"
118
-
if !indexer {
119
-
expectedFuzzy = "Union"
120
-
}
116
+
dropdownOptions := container.
117
+
Find(".menu[data-test-tag=fuzzy-dropdown]").
118
+
Find("input[type=radio][name=mode]").
119
+
Map(func(_ int, sel *goquery.Selection) string {
120
+
attr, exists := sel.Attr("value")
121
+
assert.True(t, exists)
122
+
return attr
123
+
})
121
124
122
-
fuzzyDropdown := container.Find(".ui.dropdown[data-test-tag=fuzzy-dropdown]")
123
-
actualFuzzyText := fuzzyDropdown.Find(".menu .item[data-value=true]").First().Text()
124
-
assert.EqualValues(t, expectedFuzzy, actualFuzzyText)
125
-
126
-
if fuzzyDropdown.
127
-
Find("input[name=fuzzy][value=true]").
128
-
Length() != 0 {
129
-
actualFuzzyText = fuzzyDropdown.Find("div.text").First().Text()
130
-
assert.EqualValues(t, expectedFuzzy, actualFuzzyText)
125
+
if indexer {
126
+
assert.EqualValues(t, []string{"exact", "fuzzy"}, dropdownOptions)
127
+
} else {
128
+
assert.EqualValues(t, []string{"exact", "union", "regexp"}, dropdownOptions)
131
129
}
132
130
133
131
filenames := resultFilenames(t, doc)