@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Factor out task execution and formalize permanent failures
Summary:
- Clean up a TODO about permanent failures.
- Clean up a TODO about failing tasks after too many retries.
- Clean up a TODO about testing for bad leases.
- Make the lease/retry implementation more flexible and natural.
- Make completely bogus tasks fail permanently.
- Make PhabricatorMetaMTAWorker use new `getWaitBeforeRetry()` (as intended), not hackily implement logic in `getRequiredLeaseTime()`.
- Document worker hooks for failures and retries.
- Provide coverage on everything.
Test Plan: Ran unit tests. Ran `bin/phd debug taskmaster`.
Reviewers: btrahan
Reviewed By: btrahan
CC: aran
Maniphest Tasks: T2015
Differential Revision: https://secure.phabricator.com/D3859
+452
-45
+6
src/__phutil_library_map__.php
+6
src/__phutil_library_map__.php
···
1105
1105
'PhabricatorSyntaxHighlighter' => 'infrastructure/markup/PhabricatorSyntaxHighlighter.php',
1106
1106
'PhabricatorTaskmasterDaemon' => 'infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php',
1107
1107
'PhabricatorTestCase' => 'infrastructure/testing/PhabricatorTestCase.php',
1108
+
'PhabricatorTestWorker' => 'infrastructure/daemon/workers/__tests__/PhabricatorTestWorker.php',
1108
1109
'PhabricatorTimelineCursor' => 'infrastructure/daemon/timeline/storage/PhabricatorTimelineCursor.php',
1109
1110
'PhabricatorTimelineDAO' => 'infrastructure/daemon/timeline/storage/PhabricatorTimelineDAO.php',
1110
1111
'PhabricatorTimelineEvent' => 'infrastructure/daemon/timeline/storage/PhabricatorTimelineEvent.php',
···
1144
1145
'PhabricatorWorkerArchiveTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php',
1145
1146
'PhabricatorWorkerDAO' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerDAO.php',
1146
1147
'PhabricatorWorkerLeaseQuery' => 'infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php',
1148
+
'PhabricatorWorkerPermanentFailureException' => 'infrastructure/daemon/workers/exception/PhabricatorWorkerPermanentFailureException.php',
1147
1149
'PhabricatorWorkerTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php',
1148
1150
'PhabricatorWorkerTaskData' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTaskData.php',
1149
1151
'PhabricatorWorkerTaskDetailController' => 'applications/daemon/controller/PhabricatorWorkerTaskDetailController.php',
1150
1152
'PhabricatorWorkerTaskUpdateController' => 'applications/daemon/controller/PhabricatorWorkerTaskUpdateController.php',
1153
+
'PhabricatorWorkerTestCase' => 'infrastructure/daemon/workers/__tests__/PhabricatorWorkerTestCase.php',
1151
1154
'PhabricatorXHPASTViewController' => 'applications/xhpastview/controller/PhabricatorXHPASTViewController.php',
1152
1155
'PhabricatorXHPASTViewDAO' => 'applications/xhpastview/storage/PhabricatorXHPASTViewDAO.php',
1153
1156
'PhabricatorXHPASTViewFrameController' => 'applications/xhpastview/controller/PhabricatorXHPASTViewFrameController.php',
···
2268
2271
'PhabricatorSymbolNameLinter' => 'ArcanistXHPASTLintNamingHook',
2269
2272
'PhabricatorTaskmasterDaemon' => 'PhabricatorDaemon',
2270
2273
'PhabricatorTestCase' => 'ArcanistPhutilTestCase',
2274
+
'PhabricatorTestWorker' => 'PhabricatorWorker',
2271
2275
'PhabricatorTimelineCursor' => 'PhabricatorTimelineDAO',
2272
2276
'PhabricatorTimelineDAO' => 'PhabricatorLiskDAO',
2273
2277
'PhabricatorTimelineEvent' => 'PhabricatorTimelineDAO',
···
2307
2311
'PhabricatorWorkerArchiveTask' => 'PhabricatorWorkerTask',
2308
2312
'PhabricatorWorkerDAO' => 'PhabricatorLiskDAO',
2309
2313
'PhabricatorWorkerLeaseQuery' => 'PhabricatorQuery',
2314
+
'PhabricatorWorkerPermanentFailureException' => 'Exception',
2310
2315
'PhabricatorWorkerTask' => 'PhabricatorWorkerDAO',
2311
2316
'PhabricatorWorkerTaskData' => 'PhabricatorWorkerDAO',
2312
2317
'PhabricatorWorkerTaskDetailController' => 'PhabricatorDaemonController',
2313
2318
'PhabricatorWorkerTaskUpdateController' => 'PhabricatorDaemonController',
2319
+
'PhabricatorWorkerTestCase' => 'PhabricatorTestCase',
2314
2320
'PhabricatorXHPASTViewController' => 'PhabricatorController',
2315
2321
'PhabricatorXHPASTViewDAO' => 'PhabricatorLiskDAO',
2316
2322
'PhabricatorXHPASTViewFrameController' => 'PhabricatorXHPASTViewController',
+3
-3
src/applications/metamta/PhabricatorMetaMTAWorker.php
+3
-3
src/applications/metamta/PhabricatorMetaMTAWorker.php
···
21
21
22
22
private $message;
23
23
24
-
public function getRequiredLeaseTime() {
24
+
public function getWaitBeforeRetry(PhabricatorWorkerTask $task) {
25
25
$message_id = $this->getTaskData();
26
26
27
27
$this->message = id(new PhabricatorMetaMTAMail())->loadOneWhere(
···
30
30
return null;
31
31
}
32
32
33
-
$lease_duration = max($this->message->getNextRetry() - time(), 0) + 15;
34
-
return $lease_duration;
33
+
$wait = max($this->message->getNextRetry() - time(), 0) + 15;
34
+
return $wait;
35
35
}
36
36
37
37
public function doWork() {
+5
-32
src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php
+5
-32
src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php
···
32
32
33
33
$this->log("Working on task {$id} ({$class})...");
34
34
35
-
// TODO: We should detect if we acquired a task with an expired lease
36
-
// and log about it / bump up failure count.
37
-
38
-
// TODO: We should detect if we acquired a task with an excessive
39
-
// failure count and fail it permanently.
40
-
41
-
$data = $task->getData();
42
-
try {
43
-
if (!class_exists($class) ||
44
-
!is_subclass_of($class, 'PhabricatorWorker')) {
45
-
throw new Exception(
46
-
"Task class '{$class}' does not extend PhabricatorWorker.");
47
-
}
48
-
$worker = newv($class, array($data));
49
-
50
-
$lease = $worker->getRequiredLeaseTime();
51
-
if ($lease !== null) {
52
-
$task->setLeaseDuration($lease);
53
-
}
54
-
55
-
$t_start = microtime(true);
56
-
$worker->executeTask();
57
-
$t_end = microtime(true);
58
-
59
-
$task->archiveTask(
60
-
PhabricatorWorkerArchiveTask::RESULT_SUCCESS,
61
-
(int)(1000000 * ($t_end - $t_start)));
62
-
$this->log("Task {$id} complete! Moved to archive.");
63
-
} catch (Exception $ex) {
64
-
$task->setFailureCount($task->getFailureCount() + 1);
65
-
$task->save();
66
-
35
+
$task = $task->executeTask();
36
+
$ex = $task->getExecutionException();
37
+
if ($ex) {
67
38
$this->log("Task {$id} failed!");
68
39
throw $ex;
40
+
} else {
41
+
$this->log("Task {$id} complete! Moved to archive.");
69
42
}
70
43
}
71
44
+68
-6
src/infrastructure/daemon/workers/PhabricatorWorker.php
+68
-6
src/infrastructure/daemon/workers/PhabricatorWorker.php
···
16
16
* limitations under the License.
17
17
*/
18
18
19
+
/**
20
+
* @task config Configuring Retries and Failures
21
+
*
22
+
* @group worker
23
+
*/
19
24
abstract class PhabricatorWorker {
20
25
21
26
private $data;
22
27
28
+
29
+
/* -( Configuring Retries and Failures )----------------------------------- */
30
+
31
+
32
+
/**
33
+
* Return the number of seconds this worker needs hold a lease on the task for
34
+
* while it performs work. For most tasks you can leave this at `null`, which
35
+
* will give you a short default lease (currently 60 seconds).
36
+
*
37
+
* For tasks which may take a very long time to complete, you should return
38
+
* an upper bound on the amount of time the task may require.
39
+
*
40
+
* @return int|null Number of seconds this task needs to remain leased for,
41
+
* or null for a default (currently 60 second) lease.
42
+
*
43
+
* @task config
44
+
*/
45
+
public function getRequiredLeaseTime() {
46
+
return null;
47
+
}
48
+
49
+
50
+
/**
51
+
* Return the maximum number of times this task may be retried before it
52
+
* is considered permanently failed. By default, tasks retry indefinitely. You
53
+
* can throw a @{class:PhabricatorWorkerPermanentFailureException} to cause an
54
+
* immediate permanent failure.
55
+
*
56
+
* @return int|null Number of times the task will retry before permanent
57
+
* failure. Return `null` to retry indefinitely.
58
+
*
59
+
* @task config
60
+
*/
61
+
public function getMaximumRetryCount() {
62
+
return null;
63
+
}
64
+
65
+
66
+
/**
67
+
* Return the number of seconds a task should wait after a failure before
68
+
* retrying. For most tasks you can leave this at `null`, which will give you
69
+
* a short default retry period (currently 60 seconds).
70
+
*
71
+
* @param PhabricatorWorkerTask The task itself. This object is probably
72
+
* useful mostly to examine the failure
73
+
* count if you want to implement staggered
74
+
* retries, or to examine the execution
75
+
* exception if you want to react to
76
+
* different failures in different ways.
77
+
* @param Exception The exception which caused the failure.
78
+
* @return int|null Number of seconds to wait between retries,
79
+
* or null for a default retry period
80
+
* (currently 60 seconds).
81
+
*
82
+
* @task config
83
+
*/
84
+
public function getWaitBeforeRetry(PhabricatorWorkerTask $task) {
85
+
return null;
86
+
}
87
+
88
+
abstract protected function doWork();
89
+
90
+
23
91
final public function __construct($data) {
24
92
$this->data = $data;
25
93
}
···
31
99
final public function executeTask() {
32
100
$this->doWork();
33
101
}
34
-
35
-
public function getRequiredLeaseTime() {
36
-
return null;
37
-
}
38
-
39
-
abstract protected function doWork();
40
102
41
103
final public static function scheduleTask($task_class, $data) {
42
104
return id(new PhabricatorWorkerActiveTask())
+55
src/infrastructure/daemon/workers/__tests__/PhabricatorTestWorker.php
+55
src/infrastructure/daemon/workers/__tests__/PhabricatorTestWorker.php
···
1
+
<?php
2
+
3
+
/*
4
+
* Copyright 2012 Facebook, Inc.
5
+
*
6
+
* Licensed under the Apache License, Version 2.0 (the "License");
7
+
* you may not use this file except in compliance with the License.
8
+
* You may obtain a copy of the License at
9
+
*
10
+
* http://www.apache.org/licenses/LICENSE-2.0
11
+
*
12
+
* Unless required by applicable law or agreed to in writing, software
13
+
* distributed under the License is distributed on an "AS IS" BASIS,
14
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+
* See the License for the specific language governing permissions and
16
+
* limitations under the License.
17
+
*/
18
+
19
+
final class PhabricatorTestWorker extends PhabricatorWorker {
20
+
21
+
public function getRequiredLeaseTime() {
22
+
return idx(
23
+
$this->getTaskData(),
24
+
'getRequiredLeaseTime',
25
+
parent::getRequiredLeaseTime());
26
+
}
27
+
28
+
public function getMaximumRetryCount() {
29
+
return idx(
30
+
$this->getTaskData(),
31
+
'getMaximumRetryCount',
32
+
parent::getMaximumRetryCount());
33
+
}
34
+
35
+
public function getWaitBeforeRetry(PhabricatorWorkerTask $task) {
36
+
return idx(
37
+
$this->getTaskData(),
38
+
'getWaitBeforeRetry',
39
+
parent::getWaitBeforeRetry($task));
40
+
}
41
+
42
+
protected function doWork() {
43
+
switch (idx($this->getTaskData(), 'doWork')) {
44
+
case 'fail-temporary':
45
+
throw new Exception(
46
+
"Temporary failure!");
47
+
case 'fail-permanent':
48
+
throw new PhabricatorWorkerPermanentFailureException(
49
+
"Permanent failure!");
50
+
default:
51
+
return;
52
+
}
53
+
}
54
+
55
+
}
+202
src/infrastructure/daemon/workers/__tests__/PhabricatorWorkerTestCase.php
+202
src/infrastructure/daemon/workers/__tests__/PhabricatorWorkerTestCase.php
···
1
+
<?php
2
+
3
+
/*
4
+
* Copyright 2012 Facebook, Inc.
5
+
*
6
+
* Licensed under the Apache License, Version 2.0 (the "License");
7
+
* you may not use this file except in compliance with the License.
8
+
* You may obtain a copy of the License at
9
+
*
10
+
* http://www.apache.org/licenses/LICENSE-2.0
11
+
*
12
+
* Unless required by applicable law or agreed to in writing, software
13
+
* distributed under the License is distributed on an "AS IS" BASIS,
14
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+
* See the License for the specific language governing permissions and
16
+
* limitations under the License.
17
+
*/
18
+
19
+
final class PhabricatorWorkerTestCase extends PhabricatorTestCase {
20
+
21
+
protected function getPhabricatorTestCaseConfiguration() {
22
+
return array(
23
+
self::PHABRICATOR_TESTCONFIG_BUILD_STORAGE_FIXTURES => true,
24
+
);
25
+
}
26
+
27
+
public function testLeaseTask() {
28
+
// Leasing should work.
29
+
30
+
$task = $this->scheduleTask();
31
+
32
+
$this->expectNextLease($task);
33
+
}
34
+
35
+
public function testMultipleLease() {
36
+
// We should not be able to lease a task multiple times.
37
+
38
+
$task = $this->scheduleTask();
39
+
40
+
$this->expectNextLease($task);
41
+
$this->expectNextLease(null);
42
+
}
43
+
44
+
public function testOldestFirst() {
45
+
// Older tasks should lease first, all else being equal.
46
+
47
+
$task1 = $this->scheduleTask();
48
+
$task2 = $this->scheduleTask();
49
+
50
+
$this->expectNextLease($task1)->executeTask();
51
+
$this->expectNextLease($task2);
52
+
}
53
+
54
+
public function testNewBeforeLeased() {
55
+
// Tasks not previously leased should lease before previously leased tasks.
56
+
57
+
$task1 = $this->scheduleTask();
58
+
$task2 = $this->scheduleTask();
59
+
60
+
$task1->setLeaseOwner('test');
61
+
$task1->setLeaseExpires(time() - 100000);
62
+
$task1->forceSaveWithoutLease();
63
+
64
+
$this->expectNextLease($task2)->executeTask();
65
+
$this->expectNextLease($task1);
66
+
}
67
+
68
+
69
+
public function testExecuteTask() {
70
+
$task = $this->scheduleAndExecuteTask();
71
+
72
+
$this->assertEqual(true, $task->isArchived());
73
+
$this->assertEqual(
74
+
PhabricatorWorkerArchiveTask::RESULT_SUCCESS,
75
+
$task->getResult());
76
+
}
77
+
78
+
public function testPermanentTaskFailure() {
79
+
$task = $this->scheduleAndExecuteTask(
80
+
array(
81
+
'doWork' => 'fail-permanent',
82
+
));
83
+
84
+
$this->assertEqual(true, $task->isArchived());
85
+
$this->assertEqual(
86
+
PhabricatorWorkerArchiveTask::RESULT_FAILURE,
87
+
$task->getResult());
88
+
}
89
+
90
+
public function testTemporaryTaskFailure() {
91
+
$task = $this->scheduleAndExecuteTask(
92
+
array(
93
+
'doWork' => 'fail-temporary',
94
+
));
95
+
96
+
$this->assertEqual(false, $task->isArchived());
97
+
$this->assertEqual(
98
+
true,
99
+
($task->getExecutionException() instanceof Exception));
100
+
}
101
+
102
+
public function testTooManyTaskFailures() {
103
+
// Expect temporary failures, then a permanent failure.
104
+
$task = $this->scheduleAndExecuteTask(
105
+
array(
106
+
'doWork' => 'fail-temporary',
107
+
'getMaximumRetryCount' => 3,
108
+
'getWaitBeforeRetry' => -60,
109
+
));
110
+
111
+
// Temporary...
112
+
$this->assertEqual(false, $task->isArchived());
113
+
$this->assertEqual(
114
+
true,
115
+
($task->getExecutionException() instanceof Exception));
116
+
$this->assertEqual(1, $task->getFailureCount());
117
+
118
+
// Temporary...
119
+
$task = $this->expectNextLease($task);
120
+
$task = $task->executeTask();
121
+
$this->assertEqual(false, $task->isArchived());
122
+
$this->assertEqual(
123
+
true,
124
+
($task->getExecutionException() instanceof Exception));
125
+
$this->assertEqual(2, $task->getFailureCount());
126
+
127
+
// Temporary...
128
+
$task = $this->expectNextLease($task);
129
+
$task = $task->executeTask();
130
+
$this->assertEqual(false, $task->isArchived());
131
+
$this->assertEqual(
132
+
true,
133
+
($task->getExecutionException() instanceof Exception));
134
+
$this->assertEqual(3, $task->getFailureCount());
135
+
136
+
// Temporary...
137
+
$task = $this->expectNextLease($task);
138
+
$task = $task->executeTask();
139
+
$this->assertEqual(false, $task->isArchived());
140
+
$this->assertEqual(
141
+
true,
142
+
($task->getExecutionException() instanceof Exception));
143
+
$this->assertEqual(4, $task->getFailureCount());
144
+
145
+
// Permanent.
146
+
$task = $this->expectNextLease($task);
147
+
$task = $task->executeTask();
148
+
$this->assertEqual(true, $task->isArchived());
149
+
$this->assertEqual(
150
+
PhabricatorWorkerArchiveTask::RESULT_FAILURE,
151
+
$task->getResult());
152
+
}
153
+
154
+
public function testWaitBeforeRetry() {
155
+
$task = $this->scheduleTask(
156
+
array(
157
+
'doWork' => 'fail-temporary',
158
+
'getWaitBeforeRetry' => 1000000,
159
+
));
160
+
161
+
$this->expectNextLease($task)->executeTask();
162
+
$this->expectNextLease(null);
163
+
}
164
+
165
+
public function testRequiredLeaseTime() {
166
+
$task = $this->scheduleAndExecuteTask(
167
+
array(
168
+
'getRequiredLeaseTime' => 1000000,
169
+
));
170
+
171
+
$this->assertEqual(true, ($task->getLeaseExpires() - time()) > 1000);
172
+
}
173
+
174
+
private function expectNextLease($task) {
175
+
$leased = id(new PhabricatorWorkerLeaseQuery())
176
+
->setLimit(1)
177
+
->execute();
178
+
179
+
if ($task === null) {
180
+
$this->assertEqual(0, count($leased));
181
+
return null;
182
+
} else {
183
+
$this->assertEqual(1, count($leased));
184
+
$this->assertEqual(
185
+
(int)head($leased)->getID(),
186
+
(int)$task->getID());
187
+
return head($leased);
188
+
}
189
+
}
190
+
191
+
private function scheduleAndExecuteTask(array $data = array()) {
192
+
$task = $this->scheduleTask($data);
193
+
$task = $this->expectNextLease($task);
194
+
$task = $task->executeTask();
195
+
return $task;
196
+
}
197
+
198
+
private function scheduleTask(array $data = array()) {
199
+
return PhabricatorWorker::scheduleTask('PhabricatorTestWorker', $data);
200
+
}
201
+
202
+
}
+21
src/infrastructure/daemon/workers/exception/PhabricatorWorkerPermanentFailureException.php
+21
src/infrastructure/daemon/workers/exception/PhabricatorWorkerPermanentFailureException.php
···
1
+
<?php
2
+
3
+
/*
4
+
* Copyright 2012 Facebook, Inc.
5
+
*
6
+
* Licensed under the Apache License, Version 2.0 (the "License");
7
+
* you may not use this file except in compliance with the License.
8
+
* You may obtain a copy of the License at
9
+
*
10
+
* http://www.apache.org/licenses/LICENSE-2.0
11
+
*
12
+
* Unless required by applicable law or agreed to in writing, software
13
+
* distributed under the License is distributed on an "AS IS" BASIS,
14
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+
* See the License for the specific language governing permissions and
16
+
* limitations under the License.
17
+
*/
18
+
19
+
final class PhabricatorWorkerPermanentFailureException extends Exception {
20
+
21
+
}
+5
-2
src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php
+5
-2
src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php
···
26
26
const PHASE_UNLEASED = 'unleased';
27
27
const PHASE_EXPIRED = 'expired';
28
28
29
+
const DEFAULT_LEASE_DURATION = 60; // Seconds
30
+
29
31
private $ids;
30
32
private $limit;
31
33
···
64
66
foreach ($phases as $phase) {
65
67
queryfx(
66
68
$conn_w,
67
-
'UPDATE %T SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + 15
69
+
'UPDATE %T SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + %d
68
70
%Q %Q %Q',
69
71
$task_table->getTableName(),
70
72
$lease_ownership_name,
73
+
self::DEFAULT_LEASE_DURATION,
71
74
$this->buildWhereClause($conn_w, $phase),
72
75
$this->buildOrderClause($conn_w),
73
76
$this->buildLimitClause($conn_w, $limit - $leased));
···
118
121
case self::PHASE_UNLEASED:
119
122
$where[] = 'leaseOwner IS NULL';
120
123
break;
121
-
case self::PHASE_Expired:
124
+
case self::PHASE_EXPIRED:
122
125
$where[] = 'leaseExpires < UNIX_TIMESTAMP()';
123
126
break;
124
127
default:
+77
-2
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
+77
-2
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
···
38
38
}
39
39
40
40
public function setLeaseDuration($lease_duration) {
41
+
$this->checkLease();
41
42
$server_lease_expires = $this->serverTime + $lease_duration;
42
43
$this->setLeaseExpires($server_lease_expires);
43
-
return $this->save();
44
+
45
+
// NOTE: This is primarily to allow unit tests to set negative lease
46
+
// durations so they don't have to wait around for leases to expire. We
47
+
// check that the lease is valid above.
48
+
return $this->forceSaveWithoutLease();
44
49
}
45
50
46
51
public function save() {
47
52
$this->checkLease();
53
+
return $this->forceSaveWithoutLease();
54
+
}
48
55
56
+
public function forceSaveWithoutLease() {
49
57
$is_new = !$this->getID();
50
58
if ($is_new) {
51
59
$this->failureCount = 0;
52
60
}
53
61
54
-
if ($is_new && $this->getData()) {
62
+
if ($is_new && ($this->getData() !== null)) {
55
63
$data = new PhabricatorWorkerTaskData();
56
64
$data->setData($this->getData());
57
65
$data->save();
···
100
108
101
109
return $archive;
102
110
}
111
+
112
+
public function executeTask() {
113
+
// We do this outside of the try .. catch because we don't have permission
114
+
// to release the lease otherwise.
115
+
$this->checkLease();
116
+
117
+
try {
118
+
$id = $this->getID();
119
+
$class = $this->getTaskClass();
120
+
121
+
if (!class_exists($class)) {
122
+
throw new PhabricatorWorkerPermanentFailureException(
123
+
"Task class '{$class}' does not exist!");
124
+
}
125
+
126
+
if (!is_subclass_of($class, 'PhabricatorWorker')) {
127
+
throw new PhabricatorWorkerPermanentFailureException(
128
+
"Task class '{$class}' does not extend PhabricatorWorker.");
129
+
}
130
+
131
+
$worker = newv($class, array($this->getData()));
132
+
133
+
$maximum_failures = $worker->getMaximumRetryCount();
134
+
if ($maximum_failures !== null) {
135
+
if ($this->getFailureCount() > $maximum_failures) {
136
+
throw new PhabricatorWorkerPermanentFailureException(
137
+
"Task {$id} has exceeded the maximum number of failures ".
138
+
"({$maximum_failures}).");
139
+
}
140
+
}
141
+
142
+
$lease = $worker->getRequiredLeaseTime();
143
+
if ($lease !== null) {
144
+
$this->setLeaseDuration($lease);
145
+
}
146
+
147
+
$t_start = microtime(true);
148
+
$worker->executeTask();
149
+
$t_end = microtime(true);
150
+
$duration = (int)(1000000 * ($t_end - $t_start));
151
+
152
+
$result = $this->archiveTask(
153
+
PhabricatorWorkerArchiveTask::RESULT_SUCCESS,
154
+
$duration);
155
+
} catch (PhabricatorWorkerPermanentFailureException $ex) {
156
+
$result = $this->archiveTask(
157
+
PhabricatorWorkerArchiveTask::RESULT_FAILURE,
158
+
0);
159
+
$result->setExecutionException($ex);
160
+
} catch (Exception $ex) {
161
+
$this->setExecutionException($ex);
162
+
$this->setFailureCount($this->getFailureCount() + 1);
163
+
164
+
$retry = $worker->getWaitBeforeRetry($this);
165
+
$retry = coalesce(
166
+
$retry,
167
+
PhabricatorWorkerLeaseQuery::DEFAULT_LEASE_DURATION);
168
+
169
+
// NOTE: As a side effect, this saves the object.
170
+
$this->setLeaseDuration($retry);
171
+
172
+
$result = $this;
173
+
}
174
+
175
+
return $result;
176
+
}
177
+
103
178
104
179
}
+10
src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php
+10
src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php
···
27
27
protected $dataID;
28
28
29
29
private $data;
30
+
private $executionException;
31
+
32
+
public function setExecutionException(Exception $execution_exception) {
33
+
$this->executionException = $execution_exception;
34
+
return $this;
35
+
}
36
+
37
+
public function getExecutionException() {
38
+
return $this->executionException;
39
+
}
30
40
31
41
public function setData($data) {
32
42
$this->data = $data;