loading up the forgejo repo on tangled to test page performance
Refactor markup/csv: don't read all to memory (#29760)
(cherry picked from commit e79a807a8461a73bd66146d816f635b66e198c89)
authored by
coldWater
and committed by
Earl Warren
d413a8fc
2da0628f
+55
-12
+45
-12
modules/markup/csv/csv.go
+45
-12
modules/markup/csv/csv.go
···
77
77
}
78
78
79
79
// Render implements markup.Renderer
80
-
func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
80
+
func (r Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
81
81
tmpBlock := bufio.NewWriter(output)
82
+
maxSize := setting.UI.CSV.MaxFileSize
82
83
83
-
// FIXME: don't read all to memory
84
-
rawBytes, err := io.ReadAll(input)
84
+
if maxSize == 0 {
85
+
return r.tableRender(ctx, input, tmpBlock)
86
+
}
87
+
88
+
rawBytes, err := io.ReadAll(io.LimitReader(input, maxSize+1))
89
+
if err != nil {
90
+
return err
91
+
}
92
+
93
+
if int64(len(rawBytes)) <= maxSize {
94
+
return r.tableRender(ctx, bytes.NewReader(rawBytes), tmpBlock)
95
+
}
96
+
return r.fallbackRender(io.MultiReader(bytes.NewReader(rawBytes), input), tmpBlock)
97
+
}
98
+
99
+
func (Renderer) fallbackRender(input io.Reader, tmpBlock *bufio.Writer) error {
100
+
_, err := tmpBlock.WriteString("<pre>")
85
101
if err != nil {
86
102
return err
87
103
}
88
104
89
-
if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) {
90
-
if _, err := tmpBlock.WriteString("<pre>"); err != nil {
91
-
return err
92
-
}
93
-
if _, err := tmpBlock.WriteString(html.EscapeString(string(rawBytes))); err != nil {
94
-
return err
105
+
scan := bufio.NewScanner(input)
106
+
scan.Split(bufio.ScanRunes)
107
+
for scan.Scan() {
108
+
switch scan.Text() {
109
+
case `&`:
110
+
_, err = tmpBlock.WriteString("&")
111
+
case `'`:
112
+
_, err = tmpBlock.WriteString("'") // "'" is shorter than "'" and apos was not in HTML until HTML5.
113
+
case `<`:
114
+
_, err = tmpBlock.WriteString("<")
115
+
case `>`:
116
+
_, err = tmpBlock.WriteString(">")
117
+
case `"`:
118
+
_, err = tmpBlock.WriteString(""") // """ is shorter than """.
119
+
default:
120
+
_, err = tmpBlock.Write(scan.Bytes())
95
121
}
96
-
if _, err := tmpBlock.WriteString("</pre>"); err != nil {
122
+
if err != nil {
97
123
return err
98
124
}
99
-
return tmpBlock.Flush()
100
125
}
101
126
102
-
rd, err := csv.CreateReaderAndDetermineDelimiter(ctx, bytes.NewReader(rawBytes))
127
+
_, err = tmpBlock.WriteString("</pre>")
128
+
if err != nil {
129
+
return err
130
+
}
131
+
return tmpBlock.Flush()
132
+
}
133
+
134
+
func (Renderer) tableRender(ctx *markup.RenderContext, input io.Reader, tmpBlock *bufio.Writer) error {
135
+
rd, err := csv.CreateReaderAndDetermineDelimiter(ctx, input)
103
136
if err != nil {
104
137
return err
105
138
}
+10
modules/markup/csv/csv_test.go
+10
modules/markup/csv/csv_test.go
···
4
4
package markup
5
5
6
6
import (
7
+
"bufio"
8
+
"bytes"
7
9
"strings"
8
10
"testing"
9
11
···
29
31
assert.NoError(t, err)
30
32
assert.EqualValues(t, v, buf.String())
31
33
}
34
+
35
+
t.Run("fallbackRender", func(t *testing.T) {
36
+
var buf bytes.Buffer
37
+
err := render.fallbackRender(strings.NewReader("1,<a>\n2,<b>"), bufio.NewWriter(&buf))
38
+
assert.NoError(t, err)
39
+
want := "<pre>1,<a>\n2,<b></pre>"
40
+
assert.Equal(t, want, buf.String())
41
+
})
32
42
}