@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Move Harbormaster log compression to the worker task queue
Summary: Depends on D19130. Ref T13088. Currently, when a build log is closed we compress it in the same process. Separate this out into a dedicated worker since the plan is to do a lot more work during finalization, none of which needs to happen inline during builds (or, particuarly, inline during a Conduit call for API writes in the future).
Test Plan: Ran `bin/harbormaster write-log --trace`, saw compression run inline.
Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam
Maniphest Tasks: T13088
Differential Revision: https://secure.phabricator.com/D19131
+71
-7
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
1306
1306
'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php',
1307
1307
'HarbormasterLintMessagesController' => 'applications/harbormaster/controller/HarbormasterLintMessagesController.php',
1308
1308
'HarbormasterLintPropertyView' => 'applications/harbormaster/view/HarbormasterLintPropertyView.php',
1309
+
'HarbormasterLogWorker' => 'applications/harbormaster/worker/HarbormasterLogWorker.php',
1309
1310
'HarbormasterManagementArchiveLogsWorkflow' => 'applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php',
1310
1311
'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php',
1311
1312
'HarbormasterManagementRestartWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRestartWorkflow.php',
···
6610
6611
'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'HarbormasterBuildStepImplementation',
6611
6612
'HarbormasterLintMessagesController' => 'HarbormasterController',
6612
6613
'HarbormasterLintPropertyView' => 'AphrontView',
6614
+
'HarbormasterLogWorker' => 'HarbormasterWorker',
6613
6615
'HarbormasterManagementArchiveLogsWorkflow' => 'HarbormasterManagementWorkflow',
6614
6616
'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow',
6615
6617
'HarbormasterManagementRestartWorkflow' => 'HarbormasterManagementWorkflow',
+6
-1
src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
+6
-1
src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
···
39
39
$target_id));
40
40
}
41
41
42
-
43
42
$log = HarbormasterBuildLog::initializeNewBuildLog($target);
44
43
$log->openBuildLog();
45
44
···
49
48
50
49
$content = file_get_contents('php://stdin');
51
50
$log->append($content);
51
+
52
+
echo tsprintf(
53
+
"%s\n",
54
+
pht('Write completed. Closing log...'));
55
+
56
+
PhabricatorWorker::setRunAllTasksInProcess(true);
52
57
53
58
$log->closeBuildLog();
54
59
+13
-6
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+13
-6
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
···
52
52
throw new Exception(pht('This build log is not open!'));
53
53
}
54
54
55
-
if ($this->canCompressLog()) {
56
-
$this->compressLog();
57
-
}
58
-
59
55
$start = $this->getDateCreated();
60
56
$now = PhabricatorTime::getNow();
61
57
62
-
return $this
58
+
$this
63
59
->setDuration($now - $start)
64
60
->setLive(0)
65
61
->save();
62
+
63
+
PhabricatorWorker::scheduleTask(
64
+
'HarbormasterLogWorker',
65
+
array(
66
+
'logPHID' => $this->getPHID(),
67
+
),
68
+
array(
69
+
'objectPHID' => $this->getPHID(),
70
+
));
71
+
72
+
return $this;
66
73
}
67
74
68
75
···
201
208
return implode('', $full_text);
202
209
}
203
210
204
-
private function canCompressLog() {
211
+
public function canCompressLog() {
205
212
return function_exists('gzdeflate');
206
213
}
207
214
+50
src/applications/harbormaster/worker/HarbormasterLogWorker.php
+50
src/applications/harbormaster/worker/HarbormasterLogWorker.php
···
1
+
<?php
2
+
3
+
final class HarbormasterLogWorker extends HarbormasterWorker {
4
+
5
+
protected function doWork() {
6
+
$viewer = $this->getViewer();
7
+
8
+
$data = $this->getTaskData();
9
+
$log_phid = idx($data, 'logPHID');
10
+
11
+
$log = id(new HarbormasterBuildLogQuery())
12
+
->setViewer($viewer)
13
+
->withPHIDs(array($log_phid))
14
+
->executeOne();
15
+
if (!$log) {
16
+
throw new PhabricatorWorkerPermanentFailureException(
17
+
pht('Invalid build log PHID "%s".', $log_phid));
18
+
}
19
+
20
+
$phid_key = PhabricatorHash::digestToLength($log_phid, 14);
21
+
$lock_key = "build.log({$phid_key})";
22
+
$lock = PhabricatorGlobalLock::newLock($lock_key);
23
+
24
+
try {
25
+
$lock->lock();
26
+
} catch (PhutilLockException $ex) {
27
+
throw new PhabricatorWorkerYieldException(15);
28
+
}
29
+
30
+
$caught = null;
31
+
try {
32
+
$this->finalizeBuildLog($log);
33
+
} catch (Exception $ex) {
34
+
$caught = $ex;
35
+
}
36
+
37
+
$lock->unlock();
38
+
39
+
if ($caught) {
40
+
throw $caught;
41
+
}
42
+
}
43
+
44
+
private function finalizeBuildLog(HarbormasterBuildLog $log) {
45
+
if ($log->canCompressLog()) {
46
+
$log->compressLog();
47
+
}
48
+
}
49
+
50
+
}