this repo has no description
Fix flushing when data is larger than the buffer (#2)
copyFrom and copyLinesFrom did not increment the start offset after
reading, so the same lines were copied over and over again.
authored by
Billy Keyes
and committed by
GitHub
09f3004f
bb5b8f69
+95
-2
+9
-2
gitdiff/io.go
+9
-2
gitdiff/io.go
···
147
147
return false, err
148
148
}
149
149
150
+
const (
151
+
byteBufferSize = 32 * 1024 // from io.Copy
152
+
lineBufferSize = 32
153
+
)
154
+
150
155
// copyFrom writes bytes starting from offset off in src to dst stopping at the
151
156
// end of src or at the first error. copyFrom returns the number of bytes
152
157
// written and any error.
153
158
func copyFrom(dst io.Writer, src io.ReaderAt, off int64) (written int64, err error) {
154
-
buf := make([]byte, 32*1024) // stolen from io.Copy
159
+
buf := make([]byte, byteBufferSize)
155
160
for {
156
161
nr, rerr := src.ReadAt(buf, off)
157
162
if nr > 0 {
···
167
172
err = io.ErrShortWrite
168
173
break
169
174
}
175
+
off += int64(nr)
170
176
}
171
177
if rerr != nil {
172
178
if rerr != io.EOF {
···
182
188
// the end of src or at the first error. copyLinesFrom returns the number of
183
189
// lines written and any error.
184
190
func copyLinesFrom(dst io.Writer, src LineReaderAt, off int64) (written int64, err error) {
185
-
buf := make([][]byte, 32)
191
+
buf := make([][]byte, lineBufferSize)
186
192
ReadLoop:
187
193
for {
188
194
nr, rerr := src.ReadLinesAt(buf, off)
···
201
207
break ReadLoop
202
208
}
203
209
}
210
+
off += int64(nr)
204
211
}
205
212
if rerr != nil {
206
213
if rerr != io.EOF {
+86
gitdiff/io_test.go
+86
gitdiff/io_test.go
···
4
4
"bytes"
5
5
"fmt"
6
6
"io"
7
+
"math/rand"
7
8
"testing"
8
9
)
9
10
···
114
115
})
115
116
}
116
117
}
118
+
119
+
func TestCopyFrom(t *testing.T) {
120
+
tests := map[string]struct {
121
+
Bytes int64
122
+
Offset int64
123
+
}{
124
+
"copyAll": {
125
+
Bytes: byteBufferSize / 2,
126
+
},
127
+
"copyPartial": {
128
+
Bytes: byteBufferSize / 2,
129
+
Offset: byteBufferSize / 4,
130
+
},
131
+
"copyLarge": {
132
+
Bytes: 8 * byteBufferSize,
133
+
},
134
+
}
135
+
136
+
for name, test := range tests {
137
+
t.Run(name, func(t *testing.T) {
138
+
data := make([]byte, test.Bytes)
139
+
rand.Read(data)
140
+
141
+
var dst bytes.Buffer
142
+
n, err := copyFrom(&dst, bytes.NewReader(data), test.Offset)
143
+
if err != nil {
144
+
t.Fatalf("unexpected error copying data: %v", err)
145
+
}
146
+
if n != test.Bytes-test.Offset {
147
+
t.Fatalf("incorrect number of bytes copied: expected %d, actual %d", test.Bytes-test.Offset, n)
148
+
}
149
+
150
+
expected := data[test.Offset:]
151
+
if !bytes.Equal(expected, dst.Bytes()) {
152
+
t.Fatalf("incorrect data copied:\nexpected: %v\nactual: %v", expected, dst.Bytes())
153
+
}
154
+
})
155
+
}
156
+
}
157
+
158
+
func TestCopyLinesFrom(t *testing.T) {
159
+
tests := map[string]struct {
160
+
Lines int64
161
+
Offset int64
162
+
}{
163
+
"copyAll": {
164
+
Lines: lineBufferSize / 2,
165
+
},
166
+
"copyPartial": {
167
+
Lines: lineBufferSize / 2,
168
+
Offset: lineBufferSize / 4,
169
+
},
170
+
"copyLarge": {
171
+
Lines: 8 * lineBufferSize,
172
+
},
173
+
}
174
+
175
+
const lineLength = 128
176
+
177
+
for name, test := range tests {
178
+
t.Run(name, func(t *testing.T) {
179
+
data := make([]byte, test.Lines*lineLength)
180
+
for i := range data {
181
+
data[i] = byte(32 + rand.Intn(95)) // ascii letters, numbers, symbols
182
+
if i%lineLength == lineLength-1 {
183
+
data[i] = '\n'
184
+
}
185
+
}
186
+
187
+
var dst bytes.Buffer
188
+
n, err := copyLinesFrom(&dst, &lineReaderAt{r: bytes.NewReader(data)}, test.Offset)
189
+
if err != nil {
190
+
t.Fatalf("unexpected error copying data: %v", err)
191
+
}
192
+
if n != test.Lines-test.Offset {
193
+
t.Fatalf("incorrect number of lines copied: expected %d, actual %d", test.Lines-test.Offset, n)
194
+
}
195
+
196
+
expected := data[test.Offset*lineLength:]
197
+
if !bytes.Equal(expected, dst.Bytes()) {
198
+
t.Fatalf("incorrect data copied:\nexpected: %v\nactual: %v", expected, dst.Bytes())
199
+
}
200
+
})
201
+
}
202
+
}