@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
When showing a small piece of a Harbormaster build log, load a small piece of data instead of the entire log
Summary: Depends on D19148. Ref T13088. The new rendering always executes range requests for data it needs, and we can satisfy these requests by loading the smallest number of chunks which span that range.
Test Plan: Piped 50,000 lines of Apache log into Harbormaster, viewed it in the new UI, got sensible rendering times and a reasonable amount of data actually going over the wire.
Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam
Maniphest Tasks: T13088
Differential Revision: https://secure.phabricator.com/D19149
+120
-10
+5
resources/sql/autopatches/20180228.log.01.offset.sql
+5
resources/sql/autopatches/20180228.log.01.offset.sql
+109
-8
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+109
-8
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
···
130
130
}
131
131
132
132
public function loadData($offset, $length) {
133
-
return substr($this->getLogText(), $offset, $length);
133
+
$end = ($offset + $length);
134
+
135
+
$chunks = id(new HarbormasterBuildLogChunk())->loadAllWhere(
136
+
'logID = %d AND headOffset < %d AND tailOffset >= %d
137
+
ORDER BY headOffset ASC',
138
+
$this->getID(),
139
+
$end,
140
+
$offset);
141
+
142
+
// Make sure that whatever we read out of the database is a single
143
+
// contiguous range which contains all of the requested bytes.
144
+
$ranges = array();
145
+
foreach ($chunks as $chunk) {
146
+
$ranges[] = array(
147
+
'head' => $chunk->getHeadOffset(),
148
+
'tail' => $chunk->getTailOffset(),
149
+
);
150
+
}
151
+
152
+
$ranges = isort($ranges, 'head');
153
+
$ranges = array_values($ranges);
154
+
$count = count($ranges);
155
+
for ($ii = 0; $ii < ($count - 1); $ii++) {
156
+
if ($ranges[$ii + 1]['head'] === $ranges[$ii]['tail']) {
157
+
$ranges[$ii + 1]['head'] = $ranges[$ii]['head'];
158
+
unset($ranges[$ii]);
159
+
}
160
+
}
161
+
162
+
if (count($ranges) !== 1) {
163
+
$display_ranges = array();
164
+
foreach ($ranges as $range) {
165
+
$display_ranges[] = pht(
166
+
'(%d - %d)',
167
+
$range['head'],
168
+
$range['tail']);
169
+
}
170
+
171
+
if (!$display_ranges) {
172
+
$display_ranges[] = pht('<null>');
173
+
}
174
+
175
+
throw new Exception(
176
+
pht(
177
+
'Attempt to load log bytes (%d - %d) failed: failed to '.
178
+
'load a single contiguous range. Actual ranges: %s.',
179
+
implode('; ', $display_ranges)));
180
+
}
181
+
182
+
$range = head($ranges);
183
+
if ($range['head'] > $offset || $range['tail'] < $end) {
184
+
throw new Exception(
185
+
pht(
186
+
'Attempt to load log bytes (%d - %d) failed: the loaded range '.
187
+
'(%d - %d) does not span the requested range.',
188
+
$offset,
189
+
$end,
190
+
$range['head'],
191
+
$range['tail']));
192
+
}
193
+
194
+
$parts = array();
195
+
foreach ($chunks as $chunk) {
196
+
$parts[] = $chunk->getChunkDisplayText();
197
+
}
198
+
$parts = implode('', $parts);
199
+
200
+
$chop_head = ($offset - $range['head']);
201
+
$chop_tail = ($range['tail'] - $end);
202
+
203
+
if ($chop_head) {
204
+
$parts = substr($parts, $chop_head);
205
+
}
206
+
207
+
if ($chop_tail) {
208
+
$parts = substr($parts, 0, -$chop_tail);
209
+
}
210
+
211
+
return $parts;
134
212
}
135
213
136
214
public function getReadPosition($read_offset) {
···
220
298
221
299
$this->openTransaction();
222
300
301
+
$offset = 0;
223
302
foreach ($chunks as $chunk) {
224
303
$rope->append($chunk->getChunkDisplayText());
225
304
$chunk->delete();
226
305
227
306
while ($rope->getByteLength() > $byte_limit) {
228
-
$this->writeEncodedChunk($rope, $byte_limit, $mode);
307
+
$offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode);
229
308
}
230
309
}
231
310
232
311
while ($rope->getByteLength()) {
233
-
$this->writeEncodedChunk($rope, $byte_limit, $mode);
312
+
$offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode);
234
313
}
235
314
236
315
$this
···
240
319
$this->saveTransaction();
241
320
}
242
321
243
-
private function writeEncodedChunk(PhutilRope $rope, $length, $mode) {
322
+
private function writeEncodedChunk(
323
+
PhutilRope $rope,
324
+
$offset,
325
+
$length,
326
+
$mode) {
327
+
244
328
$data = $rope->getPrefixBytes($length);
245
329
$size = strlen($data);
246
330
···
258
342
throw new Exception(pht('Unknown chunk encoding "%s"!', $mode));
259
343
}
260
344
261
-
$this->writeChunk($mode, $size, $data);
345
+
$this->writeChunk($mode, $offset, $size, $data);
262
346
263
347
$rope->removeBytesFromHead($size);
348
+
349
+
return $size;
264
350
}
265
351
266
-
private function writeChunk($encoding, $raw_size, $data) {
352
+
private function writeChunk($encoding, $offset, $raw_size, $data) {
353
+
$head_offset = $offset;
354
+
$tail_offset = $offset + $raw_size;
355
+
267
356
return id(new HarbormasterBuildLogChunk())
268
357
->setLogID($this->getID())
269
358
->setEncoding($encoding)
359
+
->setHeadOffset($head_offset)
360
+
->setTailOffset($tail_offset)
270
361
->setSize($raw_size)
271
362
->setChunk($data)
272
363
->save();
···
397
488
if ($append_id) {
398
489
queryfx(
399
490
$conn_w,
400
-
'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d',
491
+
'UPDATE %T SET
492
+
chunk = CONCAT(chunk, %B),
493
+
size = %d,
494
+
tailOffset = headOffset + %d,
495
+
WHERE
496
+
id = %d',
401
497
$chunk_table,
402
498
$append_data,
403
499
$prefix_size + $data_size,
500
+
$prefix_size + $data_size,
404
501
$append_id);
405
502
} else {
406
-
$this->writeChunk($encoding_text, $data_size, $append_data);
503
+
$this->writeChunk(
504
+
$encoding_text,
505
+
$this->getByteLength(),
506
+
$data_size,
507
+
$append_data);
407
508
}
408
509
409
510
$this->updateLineMap($append_data);
+6
-2
src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
+6
-2
src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
···
5
5
6
6
protected $logID;
7
7
protected $encoding;
8
+
protected $headOffset;
9
+
protected $tailOffset;
8
10
protected $size;
9
11
protected $chunk;
10
12
···
20
22
self::CONFIG_COLUMN_SCHEMA => array(
21
23
'logID' => 'id',
22
24
'encoding' => 'text32',
25
+
'headOffset' => 'uint64',
26
+
'tailOffset' => 'uint64',
23
27
24
28
// T6203/NULLABILITY
25
29
// Both the type and nullability of this column are crazily wrong.
···
28
32
'chunk' => 'bytes',
29
33
),
30
34
self::CONFIG_KEY_SCHEMA => array(
31
-
'key_log' => array(
32
-
'columns' => array('logID'),
35
+
'key_offset' => array(
36
+
'columns' => array('logID', 'headOffset', 'tailOffset'),
33
37
),
34
38
),
35
39
) + parent::getConfiguration();