@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add a "filePHID" to HarbormasterBuildLog and copy logs into Files during finalization
Summary: Depends on D19131. Ref T13088. During log finalization, stream the log into Files to support "Download Log", archive to Files, and API access.
Test Plan: Ran `write-log` and `rebuild-log`, saw Files objects generate with log content and appropriate permissions.
Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam
Maniphest Tasks: T13088
Differential Revision: https://secure.phabricator.com/D19132
+148
-18
+2
resources/sql/autopatches/20180222.log.01.filephid.sql
+2
resources/sql/autopatches/20180222.log.01.filephid.sql
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
1309
1309
'HarbormasterLogWorker' => 'applications/harbormaster/worker/HarbormasterLogWorker.php',
1310
1310
'HarbormasterManagementArchiveLogsWorkflow' => 'applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php',
1311
1311
'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php',
1312
+
'HarbormasterManagementRebuildLogWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php',
1312
1313
'HarbormasterManagementRestartWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRestartWorkflow.php',
1313
1314
'HarbormasterManagementUpdateWorkflow' => 'applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php',
1314
1315
'HarbormasterManagementWorkflow' => 'applications/harbormaster/management/HarbormasterManagementWorkflow.php',
···
6614
6615
'HarbormasterLogWorker' => 'HarbormasterWorker',
6615
6616
'HarbormasterManagementArchiveLogsWorkflow' => 'HarbormasterManagementWorkflow',
6616
6617
'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow',
6618
+
'HarbormasterManagementRebuildLogWorkflow' => 'HarbormasterManagementWorkflow',
6617
6619
'HarbormasterManagementRestartWorkflow' => 'HarbormasterManagementWorkflow',
6618
6620
'HarbormasterManagementUpdateWorkflow' => 'HarbormasterManagementWorkflow',
6619
6621
'HarbormasterManagementWorkflow' => 'PhabricatorManagementWorkflow',
+54
src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php
+54
src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php
···
1
+
<?php
2
+
3
+
final class HarbormasterManagementRebuildLogWorkflow
4
+
extends HarbormasterManagementWorkflow {
5
+
6
+
protected function didConstruct() {
7
+
$this
8
+
->setName('rebuild-log')
9
+
->setExamples('**rebuild-log** --id __id__ [__options__]')
10
+
->setSynopsis(
11
+
pht(
12
+
'Rebuild the file and summary for a log. This is primarily '.
13
+
'intended to make it easier to develop new log summarizers.'))
14
+
->setArguments(
15
+
array(
16
+
array(
17
+
'name' => 'id',
18
+
'param' => 'id',
19
+
'help' => pht('Log to rebuild.'),
20
+
),
21
+
));
22
+
}
23
+
24
+
public function execute(PhutilArgumentParser $args) {
25
+
$viewer = $this->getViewer();
26
+
27
+
$log_id = $args->getArg('id');
28
+
if (!$log_id) {
29
+
throw new PhutilArgumentUsageException(
30
+
pht('Choose a build log to rebuild with "--id".'));
31
+
}
32
+
33
+
$log = id(new HarbormasterBuildLogQuery())
34
+
->setViewer($viewer)
35
+
->withIDs(array($log_id))
36
+
->executeOne();
37
+
if (!$log) {
38
+
throw new PhutilArgumentUsageException(
39
+
pht(
40
+
'Unable to load build log "%s".',
41
+
$log_id));
42
+
}
43
+
44
+
PhabricatorWorker::setRunAllTasksInProcess(true);
45
+
$log->scheduleRebuild(true);
46
+
47
+
echo tsprintf(
48
+
"%s\n",
49
+
pht('Done.'));
50
+
51
+
return 0;
52
+
}
53
+
54
+
}
+5
-3
src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
+5
-3
src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
···
7
7
$this
8
8
->setName('write-log')
9
9
->setExamples('**write-log** --target __id__ [__options__]')
10
-
->setSynopsis(pht('Write a new Harbormaster build log.'))
10
+
->setSynopsis(
11
+
pht(
12
+
'Write a new Harbormaster build log. This is primarily intended '.
13
+
'to make development and testing easier.'))
11
14
->setArguments(
12
15
array(
13
16
array(
14
17
'name' => 'target',
15
18
'param' => 'id',
16
-
'help' => pht(
17
-
'Build Target ID to attach the log to.'),
19
+
'help' => pht('Build Target ID to attach the log to.'),
18
20
),
19
21
));
20
22
}
+2
src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php
+2
src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php
+11
-5
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+11
-5
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
···
9
9
protected $logType;
10
10
protected $duration;
11
11
protected $live;
12
+
protected $filePHID;
12
13
13
14
private $buildTarget = self::ATTACHABLE;
14
15
private $rope;
15
16
private $isOpen;
16
17
17
-
const CHUNK_BYTE_LIMIT = 102400;
18
+
const CHUNK_BYTE_LIMIT = 1048576;
18
19
19
20
public function __construct() {
20
21
$this->rope = new PhutilRope();
···
60
61
->setLive(0)
61
62
->save();
62
63
64
+
$this->scheduleRebuild(false);
65
+
66
+
return $this;
67
+
}
68
+
69
+
public function scheduleRebuild($force) {
63
70
PhabricatorWorker::scheduleTask(
64
71
'HarbormasterLogWorker',
65
72
array(
66
73
'logPHID' => $this->getPHID(),
74
+
'force' => $force,
67
75
),
68
76
array(
69
77
'objectPHID' => $this->getPHID(),
70
78
));
71
-
72
-
return $this;
73
79
}
74
-
75
80
76
81
protected function getConfiguration() {
77
82
return array(
···
85
90
'duration' => 'uint32?',
86
91
87
92
'live' => 'bool',
93
+
'filePHID' => 'phid?',
88
94
),
89
95
self::CONFIG_KEY_SCHEMA => array(
90
96
'key_buildtarget' => array(
···
180
186
181
187
public function newChunkIterator() {
182
188
return id(new HarbormasterBuildLogChunkIterator($this))
183
-
->setPageSize(32);
189
+
->setPageSize(8);
184
190
}
185
191
186
192
private function loadLastChunkInfo() {
+11
-1
src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php
+11
-1
src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php
···
5
5
6
6
private $log;
7
7
private $cursor;
8
+
private $asString;
8
9
9
10
private $min = 0;
10
11
private $max = PHP_INT_MAX;
···
24
25
public function setRange($min, $max) {
25
26
$this->min = (int)$min;
26
27
$this->max = (int)$max;
28
+
return $this;
29
+
}
30
+
31
+
public function setAsString($as_string) {
32
+
$this->asString = $as_string;
27
33
return $this;
28
34
}
29
35
···
43
49
$this->cursor = last($results)->getID() + 1;
44
50
}
45
51
46
-
return $results;
52
+
if ($this->asString) {
53
+
return mpull($results, 'getChunkDisplayText');
54
+
} else {
55
+
return $results;
56
+
}
47
57
}
48
58
49
59
}
+61
-9
src/applications/harbormaster/worker/HarbormasterLogWorker.php
+61
-9
src/applications/harbormaster/worker/HarbormasterLogWorker.php
···
8
8
$data = $this->getTaskData();
9
9
$log_phid = idx($data, 'logPHID');
10
10
11
-
$log = id(new HarbormasterBuildLogQuery())
12
-
->setViewer($viewer)
13
-
->withPHIDs(array($log_phid))
14
-
->executeOne();
15
-
if (!$log) {
16
-
throw new PhabricatorWorkerPermanentFailureException(
17
-
pht('Invalid build log PHID "%s".', $log_phid));
18
-
}
19
-
20
11
$phid_key = PhabricatorHash::digestToLength($log_phid, 14);
21
12
$lock_key = "build.log({$phid_key})";
22
13
$lock = PhabricatorGlobalLock::newLock($lock_key);
···
29
20
30
21
$caught = null;
31
22
try {
23
+
$log = id(new HarbormasterBuildLogQuery())
24
+
->setViewer($viewer)
25
+
->withPHIDs(array($log_phid))
26
+
->executeOne();
27
+
if (!$log) {
28
+
throw new PhabricatorWorkerPermanentFailureException(
29
+
pht(
30
+
'Invalid build log PHID "%s".',
31
+
$log_phid));
32
+
}
33
+
34
+
if ($log->getLive()) {
35
+
throw new PhabricatorWorkerPermanentFailureException(
36
+
pht(
37
+
'Log "%s" is still live. Logs can not be finalized until '.
38
+
'they have closed.',
39
+
$log_phid));
40
+
}
41
+
32
42
$this->finalizeBuildLog($log);
33
43
} catch (Exception $ex) {
34
44
$caught = $ex;
···
42
52
}
43
53
44
54
private function finalizeBuildLog(HarbormasterBuildLog $log) {
55
+
$viewer = $this->getViewer();
56
+
57
+
$data = $this->getTaskData();
58
+
$is_force = idx($data, 'force');
59
+
45
60
if ($log->canCompressLog()) {
46
61
$log->compressLog();
47
62
}
63
+
64
+
if ($is_force) {
65
+
$file_phid = $log->getFilePHID();
66
+
if ($file_phid) {
67
+
$file = id(new PhabricatorFileQuery())
68
+
->setViewer($viewer)
69
+
->withPHIDs(array($file_phid))
70
+
->executeOne();
71
+
if ($file) {
72
+
id(new PhabricatorDestructionEngine())
73
+
->destroyObject($file);
74
+
}
75
+
$log
76
+
->setFilePHID(null)
77
+
->save();
78
+
}
79
+
}
80
+
81
+
if (!$log->getFilePHID()) {
82
+
$iterator = $log->newChunkIterator()
83
+
->setAsString(true);
84
+
85
+
$source = id(new PhabricatorIteratorFileUploadSource())
86
+
->setName('harbormaster-log-'.$log->getID().'.log')
87
+
->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
88
+
->setMIMEType('application/octet-stream')
89
+
->setIterator($iterator);
90
+
91
+
$file = $source->uploadFile();
92
+
93
+
$file->attachToObject($log->getPHID());
94
+
95
+
$log
96
+
->setFilePHID($file->getPHID())
97
+
->save();
98
+
}
99
+
48
100
}
49
101
50
102
}