loading up the forgejo repo on tangled to test page performance
Fix profile render when the README.md size is larger than 1024 bytes (#25131)
Fixes https://github.com/go-gitea/gitea/issues/25094
`GetBlobContent` will only get the first 1024 bytes, if the README.md
size is larger than 1024 bytes,
We can not render the rest of them.
After this fix, we should provide the limited size to read when call
`GetBlobContent`.
After:
---------
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
authored by
yp05327
wxiaoguang
and committed by
GitHub
22a39bb9
a583c563
+116
-12
+1
-1
models/issues/pull.go
+1
-1
models/issues/pull.go
+7
-6
modules/git/blob.go
+7
-6
modules/git/blob.go
···
20
20
return b.name
21
21
}
22
22
23
-
// GetBlobContent Gets the content of the blob as raw text
24
-
func (b *Blob) GetBlobContent() (string, error) {
23
+
// GetBlobContent Gets the limited content of the blob as raw text
24
+
func (b *Blob) GetBlobContent(limit int64) (string, error) {
25
+
if limit <= 0 {
26
+
return "", nil
27
+
}
25
28
dataRc, err := b.DataAsync()
26
29
if err != nil {
27
30
return "", err
28
31
}
29
32
defer dataRc.Close()
30
-
buf := make([]byte, 1024)
31
-
n, _ := util.ReadAtMost(dataRc, buf)
32
-
buf = buf[:n]
33
-
return string(buf), nil
33
+
buf, err := util.ReadWithLimit(dataRc, int(limit))
34
+
return string(buf), err
34
35
}
35
36
36
37
// GetBlobLineCount gets line count of the blob
+38
-1
modules/util/io.go
+38
-1
modules/util/io.go
···
4
4
package util
5
5
6
6
import (
7
+
"bytes"
7
8
"errors"
8
9
"io"
9
10
)
10
11
11
12
// ReadAtMost reads at most len(buf) bytes from r into buf.
12
13
// It returns the number of bytes copied. n is only less than len(buf) if r provides fewer bytes.
13
-
// If EOF occurs while reading, err will be nil.
14
+
// If EOF or ErrUnexpectedEOF occurs while reading, err will be nil.
14
15
func ReadAtMost(r io.Reader, buf []byte) (n int, err error) {
15
16
n, err = io.ReadFull(r, buf)
16
17
if err == io.EOF || err == io.ErrUnexpectedEOF {
17
18
err = nil
18
19
}
19
20
return n, err
21
+
}
22
+
23
+
// ReadWithLimit reads at most "limit" bytes from r into buf.
24
+
// If EOF or ErrUnexpectedEOF occurs while reading, err will be nil.
25
+
func ReadWithLimit(r io.Reader, n int) (buf []byte, err error) {
26
+
return readWithLimit(r, 1024, n)
27
+
}
28
+
29
+
func readWithLimit(r io.Reader, batch, limit int) ([]byte, error) {
30
+
if limit <= batch {
31
+
buf := make([]byte, limit)
32
+
n, err := ReadAtMost(r, buf)
33
+
if err != nil {
34
+
return nil, err
35
+
}
36
+
return buf[:n], nil
37
+
}
38
+
res := bytes.NewBuffer(make([]byte, 0, batch))
39
+
bufFix := make([]byte, batch)
40
+
eof := false
41
+
for res.Len() < limit && !eof {
42
+
bufTmp := bufFix
43
+
if res.Len()+batch > limit {
44
+
bufTmp = bufFix[:limit-res.Len()]
45
+
}
46
+
n, err := io.ReadFull(r, bufTmp)
47
+
if err == io.EOF || err == io.ErrUnexpectedEOF {
48
+
eof = true
49
+
} else if err != nil {
50
+
return nil, err
51
+
}
52
+
if _, err = res.Write(bufTmp[:n]); err != nil {
53
+
return nil, err
54
+
}
55
+
}
56
+
return res.Bytes(), nil
20
57
}
21
58
22
59
// ErrNotEmpty is an error reported when there is a non-empty reader
+66
modules/util/io_test.go
+66
modules/util/io_test.go
···
1
+
// Copyright 2023 The Gitea Authors. All rights reserved.
2
+
// SPDX-License-Identifier: MIT
3
+
4
+
package util
5
+
6
+
import (
7
+
"bytes"
8
+
"errors"
9
+
"testing"
10
+
11
+
"github.com/stretchr/testify/assert"
12
+
)
13
+
14
+
type readerWithError struct {
15
+
buf *bytes.Buffer
16
+
}
17
+
18
+
func (r *readerWithError) Read(p []byte) (n int, err error) {
19
+
if r.buf.Len() < 2 {
20
+
return 0, errors.New("test error")
21
+
}
22
+
return r.buf.Read(p)
23
+
}
24
+
25
+
func TestReadWithLimit(t *testing.T) {
26
+
bs := []byte("0123456789abcdef")
27
+
28
+
// normal test
29
+
buf, err := readWithLimit(bytes.NewBuffer(bs), 5, 2)
30
+
assert.NoError(t, err)
31
+
assert.Equal(t, []byte("01"), buf)
32
+
33
+
buf, err = readWithLimit(bytes.NewBuffer(bs), 5, 5)
34
+
assert.NoError(t, err)
35
+
assert.Equal(t, []byte("01234"), buf)
36
+
37
+
buf, err = readWithLimit(bytes.NewBuffer(bs), 5, 6)
38
+
assert.NoError(t, err)
39
+
assert.Equal(t, []byte("012345"), buf)
40
+
41
+
buf, err = readWithLimit(bytes.NewBuffer(bs), 5, len(bs))
42
+
assert.NoError(t, err)
43
+
assert.Equal(t, []byte("0123456789abcdef"), buf)
44
+
45
+
buf, err = readWithLimit(bytes.NewBuffer(bs), 5, 100)
46
+
assert.NoError(t, err)
47
+
assert.Equal(t, []byte("0123456789abcdef"), buf)
48
+
49
+
// test with error
50
+
buf, err = readWithLimit(&readerWithError{bytes.NewBuffer(bs)}, 5, 10)
51
+
assert.NoError(t, err)
52
+
assert.Equal(t, []byte("0123456789"), buf)
53
+
54
+
buf, err = readWithLimit(&readerWithError{bytes.NewBuffer(bs)}, 5, 100)
55
+
assert.ErrorContains(t, err, "test error")
56
+
assert.Empty(t, buf)
57
+
58
+
// test public function
59
+
buf, err = ReadWithLimit(bytes.NewBuffer(bs), 2)
60
+
assert.NoError(t, err)
61
+
assert.Equal(t, []byte("01"), buf)
62
+
63
+
buf, err = ReadWithLimit(bytes.NewBuffer(bs), 9999999)
64
+
assert.NoError(t, err)
65
+
assert.Equal(t, []byte("0123456789abcdef"), buf)
66
+
}
+1
-1
routers/web/repo/view.go
+1
-1
routers/web/repo/view.go
···
363
363
ctx.Data["FileError"] = ctx.Locale.Tr("actions.runs.invalid_workflow_helper", workFlowErr.Error())
364
364
}
365
365
} else if util.SliceContains([]string{"CODEOWNERS", "docs/CODEOWNERS", ".gitea/CODEOWNERS"}, ctx.Repo.TreePath) {
366
-
if data, err := blob.GetBlobContent(); err == nil {
366
+
if data, err := blob.GetBlobContent(setting.UI.MaxDisplayFileSize); err == nil {
367
367
_, warnings := issue_model.GetCodeOwnersFromContent(ctx, data)
368
368
if len(warnings) > 0 {
369
369
ctx.Data["FileWarning"] = strings.Join(warnings, "\n")
+1
-1
routers/web/user/profile.go
+1
-1
routers/web/user/profile.go
+1
-1
services/repository/files/content.go
+1
-1
services/repository/files/content.go
···
203
203
} else if entry.IsLink() {
204
204
contentsResponse.Type = string(ContentTypeLink)
205
205
// The target of a symlink file is the content of the file
206
-
targetFromContent, err := entry.Blob().GetBlobContent()
206
+
targetFromContent, err := entry.Blob().GetBlobContent(1024)
207
207
if err != nil {
208
208
return nil, err
209
209
}