@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Store the Harbormaster log chunk format on the log record
Summary: Depends on D19137. Ref T13088. This allows `rebuild-log` to skip work if the chunks are already compressed. It also prepares for a future GC which is looking for "text" or "gzip" chunks to throw away in favor of archival into Files; such a GC can use this column to find collectable logs and then write "file" to it, meaning "chunks are gone, this data is only available in Files".
Test Plan: Ran migration, saw logs populate as "text". Ran `rebuild-log`, saw logs rebuild as "gzip".
Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam
Maniphest Tasks: T13088
Differential Revision: https://secure.phabricator.com/D19138
+25
-7
+2
resources/sql/autopatches/20180223.log.02.chunkformat.sql
+2
resources/sql/autopatches/20180223.log.02.chunkformat.sql
+2
resources/sql/autopatches/20180223.log.03.chunkdefault.sql
+2
resources/sql/autopatches/20180223.log.03.chunkdefault.sql
+13
-1
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+13
-1
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
···
13
13
protected $live;
14
14
protected $filePHID;
15
15
protected $byteLength;
16
+
protected $chunkFormat;
16
17
17
18
private $buildTarget = self::ATTACHABLE;
18
19
private $rope;
···
44
45
->setBuildTargetPHID($build_target->getPHID())
45
46
->setDuration(null)
46
47
->setLive(1)
47
-
->setByteLength(0);
48
+
->setByteLength(0)
49
+
->setChunkFormat(HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT);
48
50
}
49
51
50
52
public function scheduleRebuild($force) {
···
73
75
'live' => 'bool',
74
76
'filePHID' => 'phid?',
75
77
'byteLength' => 'uint64',
78
+
'chunkFormat' => 'text32',
76
79
),
77
80
self::CONFIG_KEY_SCHEMA => array(
78
81
'key_buildtarget' => array(
···
105
108
->setPageSize(8);
106
109
}
107
110
111
+
public function newDataIterator() {
112
+
return $this->newChunkIterator()
113
+
->setAsString(true);
114
+
}
115
+
108
116
private function loadLastChunkInfo() {
109
117
$chunk_table = new HarbormasterBuildLogChunk();
110
118
$conn_w = $chunk_table->establishConnection('w');
···
187
195
while ($rope->getByteLength()) {
188
196
$this->writeEncodedChunk($rope, $byte_limit, $mode);
189
197
}
198
+
199
+
$this
200
+
->setChunkFormat($mode)
201
+
->save();
190
202
191
203
$this->saveTransaction();
192
204
}
+8
-6
src/applications/harbormaster/worker/HarbormasterLogWorker.php
+8
-6
src/applications/harbormaster/worker/HarbormasterLogWorker.php
···
58
58
$is_force = idx($data, 'force');
59
59
60
60
if (!$log->getByteLength() || $is_force) {
61
-
$iterator = $log->newChunkIterator()
62
-
->setAsString(true);
61
+
$iterator = $log->newDataIterator();
63
62
64
63
$byte_length = 0;
65
64
foreach ($iterator as $block) {
66
65
$byte_length += strlen($block);
67
66
}
67
+
68
68
$log
69
69
->setByteLength($byte_length)
70
70
->save();
71
71
}
72
72
73
-
if ($log->canCompressLog()) {
74
-
$log->compressLog();
73
+
$format_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT;
74
+
if (($log->getChunkFormat() === $format_text) || $is_force) {
75
+
if ($log->canCompressLog()) {
76
+
$log->compressLog();
77
+
}
75
78
}
76
79
77
80
if ($is_force) {
···
79
82
}
80
83
81
84
if (!$log->getFilePHID()) {
82
-
$iterator = $log->newChunkIterator()
83
-
->setAsString(true);
85
+
$iterator = $log->newDataIterator();
84
86
85
87
$source = id(new PhabricatorIteratorFileUploadSource())
86
88
->setName('harbormaster-log-'.$log->getID().'.log')