@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Allow worker tasks to have priorities
Summary: Fixes T5336. Currently, `PhabricatorWorkerLeaseQuery` is basically FIFO. It makes more sense for the queue to be a priority-queue, and to assign higher priorities to alerts (email and SMS).
Test Plan: Created dummy tasks in the queue (with different priorities). Verified that the priority field was set correctly in the DB and that the priority was shown on the `/daemon/` page. Started a `PhabricatorTaskmasterDaemon` and verified that the higher priority tasks were executed before lower priority tasks.
Reviewers: epriestley, #blessed_reviewers
Reviewed By: epriestley, #blessed_reviewers
Subscribers: epriestley, Korvin
Maniphest Tasks: T5336
Differential Revision: https://secure.phabricator.com/D9871
+131
-91
+11
resources/sql/autopatches/20140711.workerpriority.sql
+11
resources/sql/autopatches/20140711.workerpriority.sql
···
1
+
ALTER TABLE {$NAMESPACE}_worker.worker_activetask
2
+
ADD COLUMN priority int unsigned NOT NULL;
3
+
4
+
ALTER TABLE {$NAMESPACE}_worker.worker_activetask
5
+
ADD KEY (leaseOwner, priority, id);
6
+
7
+
ALTER TABLE {$NAMESPACE}_worker.worker_archivetask
8
+
ADD COLUMN priority int unsigned NOT NULL;
9
+
10
+
ALTER TABLE {$NAMESPACE}_worker.worker_archivetask
11
+
ADD KEY (leaseOwner, priority, id);
+4
-1
scripts/repository/reparse.php
+4
-1
scripts/repository/reparse.php
···
267
267
268
268
if ($all_from_repo && !$force_local) {
269
269
foreach ($classes as $class) {
270
-
PhabricatorWorker::scheduleTask($class, $spec);
270
+
PhabricatorWorker::scheduleTask(
271
+
$class,
272
+
$spec,
273
+
PhabricatorWorker::PRIORITY_IMPORT);
271
274
272
275
$commit_name = 'r'.$callsign.$commit->getCommitIdentifier();
273
276
echo " Queued '{$class}' for commit '{$commit_name}'.\n";
+3
src/applications/daemon/controller/PhabricatorDaemonConsoleController.php
+3
src/applications/daemon/controller/PhabricatorDaemonConsoleController.php
···
136
136
$task->getTaskClass(),
137
137
$task->getLeaseOwner(),
138
138
$task->getLeaseExpires() - time(),
139
+
$task->getPriority(),
139
140
$task->getFailureCount(),
140
141
phutil_tag(
141
142
'a',
···
158
159
pht('Class'),
159
160
pht('Owner'),
160
161
pht('Expires'),
162
+
pht('Priority'),
161
163
pht('Failures'),
162
164
'',
163
165
));
···
167
169
'wide',
168
170
'',
169
171
'',
172
+
'n',
170
173
'n',
171
174
'action',
172
175
));
+2
-1
src/applications/diffusion/engine/DiffusionCommitHookEngine.php
+2
-1
src/applications/diffusion/engine/DiffusionCommitHookEngine.php
+2
-1
src/applications/metamta/management/PhabricatorMailManagementResendWorkflow.php
+2
-1
src/applications/metamta/management/PhabricatorMailManagementResendWorkflow.php
+2
-1
src/applications/metamta/storage/PhabricatorMetaMTAMail.php
+2
-1
src/applications/metamta/storage/PhabricatorMetaMTAMail.php
+2
-1
src/applications/search/index/PhabricatorSearchIndexer.php
+2
-1
src/applications/search/index/PhabricatorSearchIndexer.php
+40
-28
src/infrastructure/daemon/workers/PhabricatorWorker.php
+40
-28
src/infrastructure/daemon/workers/PhabricatorWorker.php
···
9
9
private static $runAllTasksInProcess = false;
10
10
private $queuedTasks = array();
11
11
12
+
const PRIORITY_ALERTS = 4000;
13
+
const PRIORITY_DEFAULT = 3000;
14
+
const PRIORITY_BULK = 2000;
15
+
const PRIORITY_IMPORT = 1000;
16
+
12
17
13
18
/* -( Configuring Retries and Failures )----------------------------------- */
14
19
···
32
37
33
38
34
39
/**
35
-
* Return the maximum number of times this task may be retried before it
36
-
* is considered permanently failed. By default, tasks retry indefinitely. You
40
+
* Return the maximum number of times this task may be retried before it is
41
+
* considered permanently failed. By default, tasks retry indefinitely. You
37
42
* can throw a @{class:PhabricatorWorkerPermanentFailureException} to cause an
38
43
* immediate permanent failure.
39
44
*
40
-
* @return int|null Number of times the task will retry before permanent
41
-
* failure. Return `null` to retry indefinitely.
45
+
* @return int|null Number of times the task will retry before permanent
46
+
* failure. Return `null` to retry indefinitely.
42
47
*
43
48
* @task config
44
49
*/
···
52
57
* retrying. For most tasks you can leave this at `null`, which will give you
53
58
* a short default retry period (currently 60 seconds).
54
59
*
55
-
* @param PhabricatorWorkerTask The task itself. This object is probably
56
-
* useful mostly to examine the failure
57
-
* count if you want to implement staggered
58
-
* retries, or to examine the execution
59
-
* exception if you want to react to
60
-
* different failures in different ways.
61
-
* @return int|null Number of seconds to wait between retries,
62
-
* or null for a default retry period
63
-
* (currently 60 seconds).
60
+
* @param PhabricatorWorkerTask The task itself. This object is probably
61
+
* useful mostly to examine the failure count
62
+
* if you want to implement staggered retries,
63
+
* or to examine the execution exception if
64
+
* you want to react to different failures in
65
+
* different ways.
66
+
* @return int|null Number of seconds to wait between retries,
67
+
* or null for a default retry period
68
+
* (currently 60 seconds).
64
69
*
65
70
* @task config
66
71
*/
···
69
74
}
70
75
71
76
abstract protected function doWork();
72
-
73
77
74
78
final public function __construct($data) {
75
79
$this->data = $data;
···
83
87
$this->doWork();
84
88
}
85
89
86
-
final public static function scheduleTask($task_class, $data) {
90
+
final public static function scheduleTask(
91
+
$task_class,
92
+
$data,
93
+
$priority = null) {
94
+
95
+
if ($priority === null) {
96
+
$priority = self::PRIORITY_DEFAULT;
97
+
}
98
+
87
99
$task = id(new PhabricatorWorkerActiveTask())
88
100
->setTaskClass($task_class)
89
-
->setData($data);
101
+
->setData($data)
102
+
->setPriority($priority);
90
103
91
104
if (self::$runAllTasksInProcess) {
92
105
// Do the work in-process.
···
96
109
try {
97
110
$worker->doWork();
98
111
foreach ($worker->getQueuedTasks() as $queued_task) {
99
-
list($queued_class, $queued_data) = $queued_task;
100
-
self::scheduleTask($queued_class, $queued_data);
112
+
list($queued_class, $queued_data, $queued_priority) = $queued_task;
113
+
self::scheduleTask($queued_class, $queued_data, $queued_priority);
101
114
}
102
115
break;
103
116
} catch (PhabricatorWorkerYieldException $ex) {
···
106
119
'In-process task "%s" yielded for %s seconds, sleeping...',
107
120
$task_class,
108
121
$ex->getDuration()));
109
-
110
122
sleep($ex->getDuration());
111
123
}
112
124
}
···
184
196
185
197
foreach ($tasks as $task) {
186
198
if ($task->getResult() != PhabricatorWorkerArchiveTask::RESULT_SUCCESS) {
187
-
throw new Exception(
188
-
pht('Task %d failed!', $task->getID()));
199
+
throw new Exception(pht('Task %d failed!', $task->getID()));
189
200
}
190
201
}
191
202
}
···
204
215
self::$runAllTasksInProcess = $all;
205
216
}
206
217
207
-
protected function log($pattern /* $args */) {
218
+
final protected function log($pattern /* , ... */) {
208
219
$console = PhutilConsole::getConsole();
209
220
$argv = func_get_args();
210
221
call_user_func_array(array($console, 'writeLog'), $argv);
···
217
228
*
218
229
* The followup task will be queued only if this task completes cleanly.
219
230
*
220
-
* @param string Task class to queue.
221
-
* @param array Data for the followup task.
231
+
* @param string Task class to queue.
232
+
* @param array Data for the followup task.
233
+
* @param int|null Priority for the followup task.
222
234
* @return this
223
235
*/
224
-
protected function queueTask($class, array $data) {
225
-
$this->queuedTasks[] = array($class, $data);
236
+
final protected function queueTask($class, array $data, $priority = null) {
237
+
$this->queuedTasks[] = array($class, $data, $priority);
226
238
return $this;
227
239
}
228
240
···
230
242
/**
231
243
* Get tasks queued as followups by @{method:queueTask}.
232
244
*
233
-
* @return list<pair<string, wild>> Queued task specifications.
245
+
* @return list<tuple<string, wild, int|null>> Queued task specifications.
234
246
*/
235
-
public function getQueuedTasks() {
247
+
final public function getQueuedTasks() {
236
248
return $this->queuedTasks;
237
249
}
238
250
+1
-2
src/infrastructure/daemon/workers/__tests__/PhabricatorTestWorker.php
+1
-2
src/infrastructure/daemon/workers/__tests__/PhabricatorTestWorker.php
···
26
26
protected function doWork() {
27
27
switch (idx($this->getTaskData(), 'doWork')) {
28
28
case 'fail-temporary':
29
-
throw new Exception(
30
-
'Temporary failure!');
29
+
throw new Exception('Temporary failure!');
31
30
case 'fail-permanent':
32
31
throw new PhabricatorWorkerPermanentFailureException(
33
32
'Permanent failure!');
+44
-25
src/infrastructure/daemon/workers/__tests__/PhabricatorWorkerTestCase.php
+44
-25
src/infrastructure/daemon/workers/__tests__/PhabricatorWorkerTestCase.php
···
9
9
}
10
10
11
11
public function testLeaseTask() {
12
-
// Leasing should work.
13
-
14
12
$task = $this->scheduleTask();
15
-
16
-
$this->expectNextLease($task);
13
+
$this->expectNextLease($task, 'Leasing should work.');
17
14
}
18
15
19
16
public function testMultipleLease() {
20
-
// We should not be able to lease a task multiple times.
21
-
22
17
$task = $this->scheduleTask();
23
18
24
19
$this->expectNextLease($task);
25
-
$this->expectNextLease(null);
20
+
$this->expectNextLease(
21
+
null,
22
+
'We should not be able to lease a task multiple times.');
26
23
}
27
24
28
25
public function testOldestFirst() {
29
-
// Older tasks should lease first, all else being equal.
30
-
31
26
$task1 = $this->scheduleTask();
32
27
$task2 = $this->scheduleTask();
33
28
34
-
$this->expectNextLease($task1);
29
+
$this->expectNextLease(
30
+
$task1,
31
+
'Older tasks should lease first, all else being equal.');
35
32
$this->expectNextLease($task2);
36
33
}
37
34
38
35
public function testNewBeforeLeased() {
39
-
// Tasks not previously leased should lease before previously leased tasks.
40
-
41
36
$task1 = $this->scheduleTask();
42
37
$task2 = $this->scheduleTask();
43
38
···
45
40
$task1->setLeaseExpires(time() - 100000);
46
41
$task1->forceSaveWithoutLease();
47
42
48
-
$this->expectNextLease($task2);
43
+
$this->expectNextLease(
44
+
$task2,
45
+
'Tasks not previously leased should lease before previously '.
46
+
'leased tasks.');
49
47
$this->expectNextLease($task1);
50
48
}
51
49
···
138
136
public function testRequiredLeaseTime() {
139
137
$task = $this->scheduleAndExecuteTask(
140
138
array(
141
-
'getRequiredLeaseTime' => 1000000,
139
+
'getRequiredLeaseTime' => 1000000,
142
140
));
143
141
144
142
$this->assertTrue(($task->getLeaseExpires() - time()) > 1000);
145
143
}
146
144
147
145
public function testLeasedIsOldestFirst() {
148
-
// Tasks which expired earlier should lease first, all else being equal.
149
-
150
146
$task1 = $this->scheduleTask();
151
147
$task2 = $this->scheduleTask();
152
148
···
158
154
$task2->setLeaseExpires(time() - 200000);
159
155
$task2->forceSaveWithoutLease();
160
156
161
-
$this->expectNextLease($task2);
157
+
$this->expectNextLease(
158
+
$task2,
159
+
'Tasks which expired earlier should lease first, all else being equal.');
162
160
$this->expectNextLease($task1);
163
161
}
164
162
165
-
private function expectNextLease($task) {
163
+
public function testLeasedIsHighestPriority() {
164
+
$task1 = $this->scheduleTask(array(), 2);
165
+
$task2 = $this->scheduleTask(array(), 1);
166
+
$task3 = $this->scheduleTask(array(), 1);
167
+
168
+
$this->expectNextLease(
169
+
$task1,
170
+
'Tasks with a higher priority should be scheduled first.');
171
+
$this->expectNextLease(
172
+
$task2,
173
+
'Tasks with the same priority should be FIFO.');
174
+
$this->expectNextLease($task3);
175
+
}
176
+
177
+
private function expectNextLease($task, $message = null) {
166
178
$leased = id(new PhabricatorWorkerLeaseQuery())
167
179
->setLimit(1)
168
180
->execute();
169
181
170
182
if ($task === null) {
171
-
$this->assertEqual(0, count($leased));
183
+
$this->assertEqual(0, count($leased), $message);
172
184
return null;
173
185
} else {
174
-
$this->assertEqual(1, count($leased));
186
+
$this->assertEqual(1, count($leased), $message);
175
187
$this->assertEqual(
176
188
(int)head($leased)->getID(),
177
-
(int)$task->getID());
189
+
(int)$task->getID(),
190
+
$message);
178
191
return head($leased);
179
192
}
180
193
}
181
194
182
-
private function scheduleAndExecuteTask(array $data = array()) {
183
-
$task = $this->scheduleTask($data);
195
+
private function scheduleAndExecuteTask(
196
+
array $data = array(),
197
+
$priority = null) {
198
+
199
+
$task = $this->scheduleTask($data, $priority);
184
200
$task = $this->expectNextLease($task);
185
201
$task = $task->executeTask();
186
202
return $task;
187
203
}
188
204
189
-
private function scheduleTask(array $data = array()) {
190
-
return PhabricatorWorker::scheduleTask('PhabricatorTestWorker', $data);
205
+
private function scheduleTask(array $data = array(), $priority = null) {
206
+
return PhabricatorWorker::scheduleTask(
207
+
'PhabricatorTestWorker',
208
+
$data,
209
+
$priority);
191
210
}
192
211
193
212
}
+6
-20
src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php
+6
-20
src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php
···
52
52
53
53
$leased = 0;
54
54
foreach ($phases as $phase) {
55
-
56
55
// NOTE: If we issue `UPDATE ... WHERE ... ORDER BY id ASC`, the query
57
56
// goes very, very slowly. The `ORDER BY` triggers this, although we get
58
57
// the same apparent results without it. Without the ORDER BY, binary
···
126
125
}
127
126
128
127
private function buildWhereClause(AphrontDatabaseConnection $conn_w, $phase) {
129
-
130
128
$where = array();
131
129
132
130
switch ($phase) {
···
141
139
}
142
140
143
141
if ($this->ids) {
144
-
$where[] = qsprintf(
145
-
$conn_w,
146
-
'id IN (%Ld)',
147
-
$this->ids);
142
+
$where[] = qsprintf($conn_w, 'id IN (%Ld)', $this->ids);
148
143
}
149
144
150
145
return $this->formatWhereClause($where);
···
162
157
163
158
switch ($phase) {
164
159
case self::PHASE_UNLEASED:
165
-
$where[] = qsprintf(
166
-
$conn_w,
167
-
'leaseOwner IS NULL');
168
-
$where[] = qsprintf(
169
-
$conn_w,
170
-
'id IN (%Ld)',
171
-
ipull($rows, 'id'));
160
+
$where[] = qsprintf($conn_w, 'leaseOwner IS NULL');
161
+
$where[] = qsprintf($conn_w, 'id IN (%Ld)', ipull($rows, 'id'));
172
162
break;
173
163
case self::PHASE_EXPIRED:
174
164
$in = array();
···
179
169
$row['id'],
180
170
$row['leaseOwner']);
181
171
}
182
-
$where[] = qsprintf(
183
-
$conn_w,
184
-
'(%Q)',
185
-
implode(' OR ', $in));
172
+
$where[] = qsprintf($conn_w, '(%Q)', implode(' OR ', $in));
186
173
break;
187
174
default:
188
175
throw new Exception("Unknown phase '{$phase}'!");
189
176
}
190
177
191
178
return $this->formatWhereClause($where);
192
-
193
179
}
194
180
195
181
private function buildOrderClause(AphrontDatabaseConnection $conn_w, $phase) {
196
182
switch ($phase) {
197
183
case self::PHASE_UNLEASED:
198
-
// When selecting new tasks, we want to consume them in roughly
199
-
// FIFO order, so we order by the task ID.
184
+
// When selecting new tasks, we want to consume them in order of
185
+
// decreasing priority (and then FIFO).
200
186
return qsprintf($conn_w, 'ORDER BY id ASC');
201
187
case self::PHASE_EXPIRED:
202
188
// When selecting failed tasks, we want to consume them in roughly
+2
-2
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
+2
-2
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
···
86
86
->setLeaseExpires($this->getLeaseExpires())
87
87
->setFailureCount($this->getFailureCount())
88
88
->setDataID($this->getDataID())
89
+
->setPriority($this->getPriority())
89
90
->setResult($result)
90
91
->setDuration($duration);
91
92
···
164
165
if ($did_succeed) {
165
166
foreach ($worker->getQueuedTasks() as $task) {
166
167
list($class, $data) = $task;
167
-
PhabricatorWorker::scheduleTask($class, $data);
168
+
PhabricatorWorker::scheduleTask($class, $data, $this->getPriority());
168
169
}
169
170
}
170
171
171
172
return $result;
172
173
}
173
-
174
174
175
175
}
+2
-2
src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php
+2
-2
src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php
···
11
11
12
12
public function save() {
13
13
if ($this->getID() === null) {
14
-
throw new Exception(
15
-
'Trying to archive a task with no ID.');
14
+
throw new Exception('Trying to archive a task with no ID.');
16
15
}
17
16
18
17
$other = new PhabricatorWorkerActiveTask();
···
57
56
->setLeaseExpires(0)
58
57
->setFailureCount(0)
59
58
->setDataID($this->getDataID())
59
+
->setPriority($this->getPriority())
60
60
->insert();
61
61
62
62
$this->setDataID(null);
+7
-6
src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php
+7
-6
src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php
···
9
9
protected $leaseExpires;
10
10
protected $failureCount;
11
11
protected $dataID;
12
+
protected $priority;
12
13
13
14
private $data;
14
15
private $executionException;
15
16
16
-
public function setExecutionException(Exception $execution_exception) {
17
+
final public function setExecutionException(Exception $execution_exception) {
17
18
$this->executionException = $execution_exception;
18
19
return $this;
19
20
}
20
21
21
-
public function getExecutionException() {
22
+
final public function getExecutionException() {
22
23
return $this->executionException;
23
24
}
24
25
25
-
public function setData($data) {
26
+
final public function setData($data) {
26
27
$this->data = $data;
27
28
return $this;
28
29
}
29
30
30
-
public function getData() {
31
+
final public function getData() {
31
32
return $this->data;
32
33
}
33
34
34
-
public function isArchived() {
35
+
final public function isArchived() {
35
36
return ($this instanceof PhabricatorWorkerArchiveTask);
36
37
}
37
38
38
-
public function getWorkerInstance() {
39
+
final public function getWorkerInstance() {
39
40
$id = $this->getID();
40
41
$class = $this->getTaskClass();
41
42