@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Compress Harbormaster build logs inline
Summary:
Ref T5822.
- After a log is closed, compress it if possible.
- Provide `bin/harbormaster archive-logs` to make it easier to change the storage format of logs.
Test Plan:
- Ran `bin/harbormaster archive-logs` on a bunch of logs, compressing and decompressing them without issues (same hashes, same decompressed size across multiple iterations).
- Ran new builds, verified logs were compressed after they closed.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T5822
Differential Revision: https://secure.phabricator.com/D15380
+280
-32
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
1114
1114
'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php',
1115
1115
'HarbormasterLintMessagesController' => 'applications/harbormaster/controller/HarbormasterLintMessagesController.php',
1116
1116
'HarbormasterLintPropertyView' => 'applications/harbormaster/view/HarbormasterLintPropertyView.php',
1117
+
'HarbormasterManagementArchiveLogsWorkflow' => 'applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php',
1117
1118
'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php',
1118
1119
'HarbormasterManagementUpdateWorkflow' => 'applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php',
1119
1120
'HarbormasterManagementWorkflow' => 'applications/harbormaster/management/HarbormasterManagementWorkflow.php',
···
5287
5288
'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'HarbormasterBuildStepImplementation',
5288
5289
'HarbormasterLintMessagesController' => 'HarbormasterController',
5289
5290
'HarbormasterLintPropertyView' => 'AphrontView',
5291
+
'HarbormasterManagementArchiveLogsWorkflow' => 'HarbormasterManagementWorkflow',
5290
5292
'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow',
5291
5293
'HarbormasterManagementUpdateWorkflow' => 'HarbormasterManagementWorkflow',
5292
5294
'HarbormasterManagementWorkflow' => 'PhabricatorManagementWorkflow',
+150
src/applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php
+150
src/applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php
···
1
+
<?php
2
+
3
+
final class HarbormasterManagementArchiveLogsWorkflow
4
+
extends HarbormasterManagementWorkflow {
5
+
6
+
protected function didConstruct() {
7
+
$this
8
+
->setName('archive-logs')
9
+
->setExamples('**archive-logs** [__options__] --mode __mode__')
10
+
->setSynopsis(pht('Compress, decompress, store or destroy build logs.'))
11
+
->setArguments(
12
+
array(
13
+
array(
14
+
'name' => 'mode',
15
+
'param' => 'mode',
16
+
'help' => pht(
17
+
'Use "plain" to remove encoding, or "compress" to compress '.
18
+
'logs.'),
19
+
),
20
+
array(
21
+
'name' => 'details',
22
+
'help' => pht(
23
+
'Show more details about operations as they are performed. '.
24
+
'Slow! But also very reassuring!'),
25
+
),
26
+
));
27
+
}
28
+
29
+
public function execute(PhutilArgumentParser $args) {
30
+
$viewer = $this->getViewer();
31
+
32
+
$mode = $args->getArg('mode');
33
+
if (!$mode) {
34
+
throw new PhutilArgumentUsageException(
35
+
pht('Choose an archival mode with --mode.'));
36
+
}
37
+
38
+
$valid_modes = array(
39
+
'plain',
40
+
'compress',
41
+
);
42
+
43
+
$valid_modes = array_fuse($valid_modes);
44
+
if (empty($valid_modes[$mode])) {
45
+
throw new PhutilArgumentUsageException(
46
+
pht(
47
+
'Unknown mode "%s". Valid modes are: %s.',
48
+
$mode,
49
+
implode(', ', $valid_modes)));
50
+
}
51
+
52
+
$log_table = new HarbormasterBuildLog();
53
+
$logs = new LiskMigrationIterator($log_table);
54
+
55
+
$show_details = $args->getArg('details');
56
+
57
+
if ($show_details) {
58
+
$total_old = 0;
59
+
$total_new = 0;
60
+
}
61
+
62
+
foreach ($logs as $log) {
63
+
echo tsprintf(
64
+
"%s\n",
65
+
pht('Processing Harbormaster build log #%d...', $log->getID()));
66
+
67
+
if ($show_details) {
68
+
$old_stats = $this->computeDetails($log);
69
+
}
70
+
71
+
switch ($mode) {
72
+
case 'plain':
73
+
$log->decompressLog();
74
+
break;
75
+
case 'compress':
76
+
$log->compressLog();
77
+
break;
78
+
}
79
+
80
+
if ($show_details) {
81
+
$new_stats = $this->computeDetails($log);
82
+
$this->printStats($old_stats, $new_stats);
83
+
84
+
$total_old += $old_stats['bytes'];
85
+
$total_new += $new_stats['bytes'];
86
+
}
87
+
}
88
+
89
+
if ($show_details) {
90
+
echo tsprintf(
91
+
"%s\n",
92
+
pht(
93
+
'Done. Total byte size of affected logs: %s -> %s.',
94
+
new PhutilNumber($total_old),
95
+
new PhutilNumber($total_new)));
96
+
}
97
+
98
+
return 0;
99
+
}
100
+
101
+
private function computeDetails(HarbormasterBuildLog $log) {
102
+
$bytes = 0;
103
+
$chunks = 0;
104
+
$hash = hash_init('sha1');
105
+
106
+
foreach ($log->newChunkIterator() as $chunk) {
107
+
$bytes += strlen($chunk->getChunk());
108
+
$chunks++;
109
+
hash_update($hash, $chunk->getChunkDisplayText());
110
+
}
111
+
112
+
return array(
113
+
'bytes' => $bytes,
114
+
'chunks' => $chunks,
115
+
'hash' => hash_final($hash),
116
+
);
117
+
}
118
+
119
+
private function printStats(array $old_stats, array $new_stats) {
120
+
echo tsprintf(
121
+
" %s\n",
122
+
pht(
123
+
'%s: %s -> %s',
124
+
pht('Stored Bytes'),
125
+
new PhutilNumber($old_stats['bytes']),
126
+
new PhutilNumber($new_stats['bytes'])));
127
+
128
+
echo tsprintf(
129
+
" %s\n",
130
+
pht(
131
+
'%s: %s -> %s',
132
+
pht('Stored Chunks'),
133
+
new PhutilNumber($old_stats['chunks']),
134
+
new PhutilNumber($new_stats['chunks'])));
135
+
136
+
echo tsprintf(
137
+
" %s\n",
138
+
pht(
139
+
'%s: %s -> %s',
140
+
pht('Data Hash'),
141
+
$old_stats['hash'],
142
+
$new_stats['hash']));
143
+
144
+
if ($old_stats['hash'] !== $new_stats['hash']) {
145
+
throw new Exception(
146
+
pht('Log data hashes differ! Something is tragically wrong!'));
147
+
}
148
+
}
149
+
150
+
}
+101
-25
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+101
-25
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
···
52
52
throw new Exception(pht('This build log is not open!'));
53
53
}
54
54
55
-
// TODO: Encode the log contents in a gzipped format.
56
-
57
-
$this->reload();
55
+
if ($this->canCompressLog()) {
56
+
$this->compressLog();
57
+
}
58
58
59
59
$start = $this->getDateCreated();
60
60
$now = PhabricatorTime::getNow();
···
135
135
}
136
136
137
137
$conn_w = $this->establishConnection('w');
138
-
$tail = queryfx_one(
139
-
$conn_w,
140
-
'SELECT id, size, encoding FROM %T WHERE logID = %d
141
-
ORDER BY id DESC LIMIT 1',
142
-
$chunk_table,
143
-
$this->getID());
138
+
$last = $this->loadLastChunkInfo();
144
139
145
140
$can_append =
146
-
($tail) &&
147
-
($tail['encoding'] == $encoding_text) &&
148
-
($tail['size'] < $chunk_limit);
141
+
($last) &&
142
+
($last['encoding'] == $encoding_text) &&
143
+
($last['size'] < $chunk_limit);
149
144
if ($can_append) {
150
-
$append_id = $tail['id'];
151
-
$prefix_size = $tail['size'];
145
+
$append_id = $last['id'];
146
+
$prefix_size = $last['size'];
152
147
} else {
153
148
$append_id = null;
154
149
$prefix_size = 0;
···
167
162
$prefix_size + $data_size,
168
163
$append_id);
169
164
} else {
170
-
queryfx(
171
-
$conn_w,
172
-
'INSERT INTO %T (logID, encoding, size, chunk)
173
-
VALUES (%d, %s, %d, %B)',
174
-
$chunk_table,
175
-
$this->getID(),
176
-
$encoding_text,
177
-
$data_size,
178
-
$append_data);
165
+
$this->writeChunk($encoding_text, $data_size, $append_data);
179
166
}
180
167
181
-
$rope->removeBytesFromHead(strlen($append_data));
168
+
$rope->removeBytesFromHead($data_size);
182
169
}
183
170
}
184
171
185
172
public function newChunkIterator() {
186
-
return new HarbormasterBuildLogChunkIterator($this);
173
+
return id(new HarbormasterBuildLogChunkIterator($this))
174
+
->setPageSize(32);
175
+
}
176
+
177
+
private function loadLastChunkInfo() {
178
+
$chunk_table = new HarbormasterBuildLogChunk();
179
+
$conn_w = $chunk_table->establishConnection('w');
180
+
181
+
return queryfx_one(
182
+
$conn_w,
183
+
'SELECT id, size, encoding FROM %T WHERE logID = %d
184
+
ORDER BY id DESC LIMIT 1',
185
+
$chunk_table->getTableName(),
186
+
$this->getID());
187
187
}
188
188
189
189
public function getLogText() {
···
197
197
}
198
198
199
199
return implode('', $full_text);
200
+
}
201
+
202
+
private function canCompressLog() {
203
+
return function_exists('gzdeflate');
204
+
}
205
+
206
+
public function compressLog() {
207
+
$this->processLog(HarbormasterBuildLogChunk::CHUNK_ENCODING_GZIP);
208
+
}
209
+
210
+
public function decompressLog() {
211
+
$this->processLog(HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT);
212
+
}
213
+
214
+
private function processLog($mode) {
215
+
$chunks = $this->newChunkIterator();
216
+
217
+
// NOTE: Because we're going to insert new chunks, we need to stop the
218
+
// iterator once it hits the final chunk which currently exists. Otherwise,
219
+
// it may start consuming chunks we just wrote and run forever.
220
+
$last = $this->loadLastChunkInfo();
221
+
if ($last) {
222
+
$chunks->setRange(null, $last['id']);
223
+
}
224
+
225
+
$byte_limit = self::CHUNK_BYTE_LIMIT;
226
+
$rope = new PhutilRope();
227
+
228
+
$this->openTransaction();
229
+
230
+
foreach ($chunks as $chunk) {
231
+
$rope->append($chunk->getChunkDisplayText());
232
+
$chunk->delete();
233
+
234
+
while ($rope->getByteLength() > $byte_limit) {
235
+
$this->writeEncodedChunk($rope, $byte_limit, $mode);
236
+
}
237
+
}
238
+
239
+
while ($rope->getByteLength()) {
240
+
$this->writeEncodedChunk($rope, $byte_limit, $mode);
241
+
}
242
+
243
+
$this->saveTransaction();
244
+
}
245
+
246
+
private function writeEncodedChunk(PhutilRope $rope, $length, $mode) {
247
+
$data = $rope->getPrefixBytes($length);
248
+
$size = strlen($data);
249
+
250
+
switch ($mode) {
251
+
case HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT:
252
+
// Do nothing.
253
+
break;
254
+
case HarbormasterBuildLogChunk::CHUNK_ENCODING_GZIP:
255
+
$data = gzdeflate($data);
256
+
if ($data === false) {
257
+
throw new Exception(pht('Failed to gzdeflate() log data!'));
258
+
}
259
+
break;
260
+
default:
261
+
throw new Exception(pht('Unknown chunk encoding "%s"!', $mode));
262
+
}
263
+
264
+
$this->writeChunk($mode, $size, $data);
265
+
266
+
$rope->removeBytesFromHead($size);
267
+
}
268
+
269
+
private function writeChunk($encoding, $raw_size, $data) {
270
+
return id(new HarbormasterBuildLogChunk())
271
+
->setLogID($this->getID())
272
+
->setEncoding($encoding)
273
+
->setSize($raw_size)
274
+
->setChunk($data)
275
+
->save();
200
276
}
201
277
202
278
+10
-4
src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
+10
-4
src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
···
8
8
protected $size;
9
9
protected $chunk;
10
10
11
-
12
-
/**
13
-
* The log is encoded as plain text.
14
-
*/
15
11
const CHUNK_ENCODING_TEXT = 'text';
12
+
const CHUNK_ENCODING_GZIP = 'gzip';
16
13
17
14
protected function getConfiguration() {
18
15
return array(
19
16
self::CONFIG_TIMESTAMPS => false,
17
+
self::CONFIG_BINARY => array(
18
+
'chunk' => true,
19
+
),
20
20
self::CONFIG_COLUMN_SCHEMA => array(
21
21
'logID' => 'id',
22
22
'encoding' => 'text32',
···
42
42
switch ($encoding) {
43
43
case self::CHUNK_ENCODING_TEXT:
44
44
// Do nothing, data is already plaintext.
45
+
break;
46
+
case self::CHUNK_ENCODING_GZIP:
47
+
$data = gzinflate($data);
48
+
if ($data === false) {
49
+
throw new Exception(pht('Unable to inflate log chunk!'));
50
+
}
45
51
break;
46
52
default:
47
53
throw new Exception(
+17
-3
src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php
+17
-3
src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php
···
6
6
private $log;
7
7
private $cursor;
8
8
9
+
private $min = 0;
10
+
private $max = PHP_INT_MAX;
11
+
9
12
public function __construct(HarbormasterBuildLog $log) {
10
13
$this->log = $log;
11
14
}
12
15
13
16
protected function didRewind() {
14
-
$this->cursor = 0;
17
+
$this->cursor = $this->min;
15
18
}
16
19
17
20
public function key() {
18
21
return $this->current()->getID();
19
22
}
20
23
24
+
public function setRange($min, $max) {
25
+
$this->min = (int)$min;
26
+
$this->max = (int)$max;
27
+
return $this;
28
+
}
29
+
21
30
protected function loadPage() {
31
+
if ($this->cursor > $this->max) {
32
+
return array();
33
+
}
34
+
22
35
$results = id(new HarbormasterBuildLogChunk())->loadAllWhere(
23
-
'logID = %d AND id > %d ORDER BY id ASC LIMIT %d',
36
+
'logID = %d AND id >= %d AND id <= %d ORDER BY id ASC LIMIT %d',
24
37
$this->log->getID(),
25
38
$this->cursor,
39
+
$this->max,
26
40
$this->getPageSize());
27
41
28
42
if ($results) {
29
-
$this->cursor = last($results)->getID();
43
+
$this->cursor = last($results)->getID() + 1;
30
44
}
31
45
32
46
return $results;