@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Implement an iterator for build log chunks
Summary: Ref T5822. This will make it easier to compress and archive chunks without needing to hold them in memory.
Test Plan: Ran a build, looked at some logs.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T5822
Differential Revision: https://secure.phabricator.com/D15378
+75
-26
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
1045
1045
'HarbormasterBuildLintMessage' => 'applications/harbormaster/storage/build/HarbormasterBuildLintMessage.php',
1046
1046
'HarbormasterBuildLog' => 'applications/harbormaster/storage/build/HarbormasterBuildLog.php',
1047
1047
'HarbormasterBuildLogChunk' => 'applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php',
1048
+
'HarbormasterBuildLogChunkIterator' => 'applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php',
1048
1049
'HarbormasterBuildLogPHIDType' => 'applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php',
1049
1050
'HarbormasterBuildLogQuery' => 'applications/harbormaster/query/HarbormasterBuildLogQuery.php',
1050
1051
'HarbormasterBuildMessage' => 'applications/harbormaster/storage/HarbormasterBuildMessage.php',
···
5193
5194
'PhabricatorPolicyInterface',
5194
5195
),
5195
5196
'HarbormasterBuildLogChunk' => 'HarbormasterDAO',
5197
+
'HarbormasterBuildLogChunkIterator' => 'PhutilBufferedIterator',
5196
5198
'HarbormasterBuildLogPHIDType' => 'PhabricatorPHIDType',
5197
5199
'HarbormasterBuildLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
5198
5200
'HarbormasterBuildMessage' => array(
+15
-26
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+15
-26
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
···
16
16
17
17
const CHUNK_BYTE_LIMIT = 102400;
18
18
19
-
/**
20
-
* The log is encoded as plain text.
21
-
*/
22
-
const ENCODING_TEXT = 'text';
23
-
24
19
public function __construct() {
25
20
$this->rope = new PhutilRope();
26
21
}
···
129
124
130
125
$chunk_table = id(new HarbormasterBuildLogChunk())->getTableName();
131
126
$chunk_limit = self::CHUNK_BYTE_LIMIT;
127
+
$encoding_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT;
128
+
132
129
$rope = $this->rope;
133
130
134
131
while (true) {
···
147
144
148
145
$can_append =
149
146
($tail) &&
150
-
($tail['encoding'] == self::ENCODING_TEXT) &&
147
+
($tail['encoding'] == $encoding_text) &&
151
148
($tail['size'] < $chunk_limit);
152
149
if ($can_append) {
153
150
$append_id = $tail['id'];
···
176
173
VALUES (%d, %s, %d, %B)',
177
174
$chunk_table,
178
175
$this->getID(),
179
-
self::ENCODING_TEXT,
176
+
$encoding_text,
180
177
$data_size,
181
178
$append_data);
182
179
}
···
185
182
}
186
183
}
187
184
185
+
public function newChunkIterator() {
186
+
return new HarbormasterBuildLogChunkIterator($this);
187
+
}
188
+
188
189
public function getLogText() {
189
-
// TODO: This won't cope very well if we're pulling like a 700MB
190
-
// log file out of the DB. We should probably implement some sort
191
-
// of optional limit parameter so that when we're rendering out only
192
-
// 25 lines in the UI, we don't wastefully read in the whole log.
190
+
// TODO: Remove this method since it won't scale for big logs.
193
191
194
-
// We have to read our content out of the database and stitch all of
195
-
// the log data back together.
196
-
$conn = $this->establishConnection('r');
197
-
$result = queryfx_all(
198
-
$conn,
199
-
'SELECT chunk '.
200
-
'FROM %T '.
201
-
'WHERE logID = %d '.
202
-
'ORDER BY id ASC',
203
-
id(new HarbormasterBuildLogChunk())->getTableName(),
204
-
$this->getID());
192
+
$all_chunks = $this->newChunkIterator();
205
193
206
-
$content = '';
207
-
foreach ($result as $row) {
208
-
$content .= $row['chunk'];
194
+
$full_text = array();
195
+
foreach ($all_chunks as $chunk) {
196
+
$full_text[] = $chunk->getChunkDisplayText();
209
197
}
210
-
return $content;
198
+
199
+
return implode('', $full_text);
211
200
}
212
201
213
202
+23
src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
+23
src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
···
8
8
protected $size;
9
9
protected $chunk;
10
10
11
+
12
+
/**
13
+
* The log is encoded as plain text.
14
+
*/
15
+
const CHUNK_ENCODING_TEXT = 'text';
16
+
11
17
protected function getConfiguration() {
12
18
return array(
13
19
self::CONFIG_TIMESTAMPS => false,
···
28
34
),
29
35
) + parent::getConfiguration();
30
36
}
37
+
38
+
public function getChunkDisplayText() {
39
+
$data = $this->getChunk();
40
+
$encoding = $this->getEncoding();
41
+
42
+
switch ($encoding) {
43
+
case self::CHUNK_ENCODING_TEXT:
44
+
// Do nothing, data is already plaintext.
45
+
break;
46
+
default:
47
+
throw new Exception(
48
+
pht('Unknown log chunk encoding ("%s")!', $encoding));
49
+
}
50
+
51
+
return $data;
52
+
}
53
+
31
54
32
55
}
+35
src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php
+35
src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php
···
1
+
<?php
2
+
3
+
final class HarbormasterBuildLogChunkIterator
4
+
extends PhutilBufferedIterator {
5
+
6
+
private $log;
7
+
private $cursor;
8
+
9
+
public function __construct(HarbormasterBuildLog $log) {
10
+
$this->log = $log;
11
+
}
12
+
13
+
protected function didRewind() {
14
+
$this->cursor = 0;
15
+
}
16
+
17
+
public function key() {
18
+
return $this->current()->getID();
19
+
}
20
+
21
+
protected function loadPage() {
22
+
$results = id(new HarbormasterBuildLogChunk())->loadAllWhere(
23
+
'logID = %d AND id > %d ORDER BY id ASC LIMIT %d',
24
+
$this->log->getID(),
25
+
$this->cursor,
26
+
$this->getPageSize());
27
+
28
+
if ($results) {
29
+
$this->cursor = last($results)->getID();
30
+
}
31
+
32
+
return $results;
33
+
}
34
+
35
+
}