loading up the forgejo repo on tangled to test page performance
Merge pull request '[BUG] Handle bigger files in `git grep`' (#3160) from gusted/forgejo-grep into forgejo
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/3160
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Reviewed-by: 0ko <0ko@noreply.codeberg.org>
Reviewed-by: Shiny Nematoda <snematoda@noreply.codeberg.org>
+43
-4
+16
-4
modules/git/grep.go
+16
-4
modules/git/grep.go
···
10
10
"context"
11
11
"errors"
12
12
"fmt"
13
+
"io"
13
14
"os"
14
15
"strconv"
15
16
"strings"
···
80
81
defer stdoutReader.Close()
81
82
82
83
isInBlock := false
83
-
scanner := bufio.NewScanner(stdoutReader)
84
+
scanner := bufio.NewReader(stdoutReader)
84
85
var res *GrepResult
85
-
for scanner.Scan() {
86
-
line := scanner.Text()
86
+
for {
87
+
line, err := scanner.ReadString('\n')
88
+
if err != nil {
89
+
if err == io.EOF {
90
+
return nil
91
+
}
92
+
return err
93
+
}
94
+
// Remove delimiter.
95
+
if len(line) > 0 {
96
+
line = line[:len(line)-1]
97
+
}
98
+
87
99
if !isInBlock {
88
100
if _ /* ref */, filename, ok := strings.Cut(line, ":"); ok {
89
101
isInBlock = true
···
109
121
res.LineCodes = append(res.LineCodes, lineCode)
110
122
}
111
123
}
112
-
return scanner.Err()
124
+
return nil
113
125
},
114
126
})
115
127
// git grep exits by cancel (killed), usually it is caused by the limit of results
+27
modules/git/grep_test.go
+27
modules/git/grep_test.go
···
4
4
package git
5
5
6
6
import (
7
+
"bytes"
7
8
"context"
9
+
"os"
10
+
"path"
8
11
"path/filepath"
9
12
"testing"
10
13
···
49
52
assert.Error(t, err)
50
53
assert.Len(t, res, 0)
51
54
}
55
+
56
+
func TestGrepLongFiles(t *testing.T) {
57
+
tmpDir := t.TempDir()
58
+
59
+
err := InitRepository(DefaultContext, tmpDir, false, Sha1ObjectFormat.Name())
60
+
assert.NoError(t, err)
61
+
62
+
gitRepo, err := openRepositoryWithDefaultContext(tmpDir)
63
+
assert.NoError(t, err)
64
+
defer gitRepo.Close()
65
+
66
+
assert.NoError(t, os.WriteFile(path.Join(tmpDir, "README.md"), bytes.Repeat([]byte{'a'}, 65*1024), 0o666))
67
+
68
+
err = AddChanges(tmpDir, true)
69
+
assert.NoError(t, err)
70
+
71
+
err = CommitChanges(tmpDir, CommitChangesOptions{Message: "Long file"})
72
+
assert.NoError(t, err)
73
+
74
+
res, err := GrepSearch(context.Background(), gitRepo, "a", GrepOptions{})
75
+
assert.NoError(t, err)
76
+
assert.Len(t, res, 1)
77
+
assert.Len(t, res[0].LineCodes[0], 65*1024)
78
+
}