@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Make task queue more robust against long-running tasks
Summary:
See discussion in D8773. Three small adjustments which should help prevent this kind of issue:
- When queueing followup tasks, hold them on the worker until we finish the task, then queue them only if the work was successful.
- Increase the default lease time from 60 seconds to 2 hours. Although most tasks finish in far fewer than 60 seconds, the daemons are generally stable nowadays and these short leases don't serve much of a purpose. I think they also date from an era where lease expiry and failure were less clearly distinguished.
- Increase the default wait-after-failure from 60 seconds to 5 minutes. This largely dates from the MetaMTA era, where Facebook ran services with high failure rates and it was appropriate to repeatedly hammer them until things went through. In modern infrastructure, such failures are rare.
Test Plan:
- Verified that tasks queued properly after the main task was updated.
- Verified that leases default to 7200 seconds.
- Intentionally failed a task and verified default 300 second wait before retry.
- Removed all default leases shorter than 7200 seconds (there was only one).
- Checked all the wait before retry implementations for anything much shorter than 5 minutes (they all seem reasonable).
Reviewers: btrahan, sowedance
Reviewed By: sowedance
Subscribers: epriestley
Differential Revision: https://secure.phabricator.com/D8774
+59
-21
+2
-2
src/applications/feed/worker/FeedPublisherWorker.php
+2
-2
src/applications/feed/worker/FeedPublisherWorker.php
···
7
7
8
8
$uris = PhabricatorEnv::getEnvConfig('feed.http-hooks');
9
9
foreach ($uris as $uri) {
10
-
PhabricatorWorker::scheduleTask(
10
+
$this->queueTask(
11
11
'FeedPublisherHTTPWorker',
12
12
array(
13
13
'key' => $story->getChronologicalKey(),
···
27
27
if (!$worker->isEnabled()) {
28
28
continue;
29
29
}
30
-
PhabricatorWorker::scheduleTask(
30
+
$this->queueTask(
31
31
get_class($worker),
32
32
array(
33
33
'key' => $story->getChronologicalKey(),
+1
-1
src/applications/harbormaster/worker/HarbormasterTargetWorker.php
+1
-1
src/applications/harbormaster/worker/HarbormasterTargetWorker.php
+1
-1
src/applications/repository/worker/PhabricatorRepositoryCommitHeraldWorker.php
+1
-1
src/applications/repository/worker/PhabricatorRepositoryCommitHeraldWorker.php
+1
-6
src/applications/repository/worker/PhabricatorRepositoryCommitOwnersWorker.php
+1
-6
src/applications/repository/worker/PhabricatorRepositoryCommitOwnersWorker.php
···
3
3
final class PhabricatorRepositoryCommitOwnersWorker
4
4
extends PhabricatorRepositoryCommitParserWorker {
5
5
6
-
public function getRequiredLeaseTime() {
7
-
// Commit owner parser may take a while to process for huge commits.
8
-
return 60 * 60;
9
-
}
10
-
11
6
protected function parseCommit(
12
7
PhabricatorRepository $repository,
13
8
PhabricatorRepositoryCommit $commit) {
···
18
13
PhabricatorRepositoryCommit::IMPORTED_OWNERS);
19
14
20
15
if ($this->shouldQueueFollowupTasks()) {
21
-
PhabricatorWorker::scheduleTask(
16
+
$this->queueTask(
22
17
'PhabricatorRepositoryCommitHeraldWorker',
23
18
array(
24
19
'commitID' => $commit->getID(),
+2
-2
src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php
+2
-2
src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php
···
6
6
public function getRequiredLeaseTime() {
7
7
// It can take a very long time to parse commits; some commits in the
8
8
// Facebook repository affect many millions of paths. Acquire 24h leases.
9
-
return 60 * 60 * 24;
9
+
return phutil_units('24 hours in seconds');
10
10
}
11
11
12
12
abstract protected function parseCommitChanges(
···
98
98
99
99
PhabricatorOwnersPackagePathValidator::updateOwnersPackagePaths($commit);
100
100
if ($this->shouldQueueFollowupTasks()) {
101
-
PhabricatorWorker::scheduleTask(
101
+
$this->queueTask(
102
102
'PhabricatorRepositoryCommitOwnersWorker',
103
103
array(
104
104
'commitID' => $commit->getID(),
+1
-1
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryGitCommitMessageParserWorker.php
+1
-1
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryGitCommitMessageParserWorker.php
+1
-1
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryMercurialCommitMessageParserWorker.php
+1
-1
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryMercurialCommitMessageParserWorker.php
+1
-1
src/applications/repository/worker/commitmessageparser/PhabricatorRepositorySvnCommitMessageParserWorker.php
+1
-1
src/applications/repository/worker/commitmessageparser/PhabricatorRepositorySvnCommitMessageParserWorker.php
+28
-2
src/infrastructure/daemon/workers/PhabricatorWorker.php
+28
-2
src/infrastructure/daemon/workers/PhabricatorWorker.php
···
9
9
10
10
private $data;
11
11
private static $runAllTasksInProcess = false;
12
+
private $queuedTasks = array();
12
13
13
14
14
15
/* -( Configuring Retries and Failures )----------------------------------- */
···
17
18
/**
18
19
* Return the number of seconds this worker needs hold a lease on the task for
19
20
* while it performs work. For most tasks you can leave this at `null`, which
20
-
* will give you a short default lease (currently 60 seconds).
21
+
* will give you a default lease (currently 2 hours).
21
22
*
22
23
* For tasks which may take a very long time to complete, you should return
23
24
* an upper bound on the amount of time the task may require.
24
25
*
25
26
* @return int|null Number of seconds this task needs to remain leased for,
26
-
* or null for a default (currently 60 second) lease.
27
+
* or null for a default lease.
27
28
*
28
29
* @task config
29
30
*/
···
192
193
$argv = func_get_args();
193
194
call_user_func_array(array($console, 'writeLog'), $argv);
194
195
return $this;
196
+
}
197
+
198
+
199
+
/**
200
+
* Queue a task to be executed after this one suceeds.
201
+
*
202
+
* The followup task will be queued only if this task completes cleanly.
203
+
*
204
+
* @param string Task class to queue.
205
+
* @param array Data for the followup task.
206
+
* @return this
207
+
*/
208
+
protected function queueTask($class, array $data) {
209
+
$this->queuedTasks[] = array($class, $data);
210
+
return $this;
211
+
}
212
+
213
+
214
+
/**
215
+
* Get tasks queued as followups by @{method:queueTask}.
216
+
*
217
+
* @return list<pair<string, wild>> Queued task specifications.
218
+
*/
219
+
public function getQueuedTasks() {
220
+
return $this->queuedTasks;
195
221
}
196
222
197
223
}
+9
-3
src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php
+9
-3
src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php
···
10
10
const PHASE_UNLEASED = 'unleased';
11
11
const PHASE_EXPIRED = 'expired';
12
12
13
-
const DEFAULT_LEASE_DURATION = 60; // Seconds
14
-
15
13
private $ids;
16
14
private $limit;
15
+
16
+
public static function getDefaultWaitBeforeRetry() {
17
+
return phutil_units('5 minutes in seconds');
18
+
}
19
+
20
+
public static function getDefaultLeaseDuration() {
21
+
return phutil_units('2 hours in seconds');
22
+
}
17
23
18
24
public function withIDs(array $ids) {
19
25
$this->ids = $ids;
···
78
84
%Q',
79
85
$task_table->getTableName(),
80
86
$lease_ownership_name,
81
-
self::DEFAULT_LEASE_DURATION,
87
+
self::getDefaultLeaseDuration(),
82
88
$this->buildUpdateWhereClause($conn_w, $phase, $rows));
83
89
84
90
$leased += $conn_w->getAffectedRows();
+12
-1
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
+12
-1
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
···
100
100
// to release the lease otherwise.
101
101
$this->checkLease();
102
102
103
+
$did_succeed = false;
103
104
try {
104
105
$worker = $this->getWorkerInstance();
105
106
···
126
127
$result = $this->archiveTask(
127
128
PhabricatorWorkerArchiveTask::RESULT_SUCCESS,
128
129
$duration);
130
+
$did_succeed = true;
129
131
} catch (PhabricatorWorkerPermanentFailureException $ex) {
130
132
$result = $this->archiveTask(
131
133
PhabricatorWorkerArchiveTask::RESULT_FAILURE,
···
139
141
$retry = $worker->getWaitBeforeRetry($this);
140
142
$retry = coalesce(
141
143
$retry,
142
-
PhabricatorWorkerLeaseQuery::DEFAULT_LEASE_DURATION);
144
+
PhabricatorWorkerLeaseQuery::getDefaultWaitBeforeRetry());
143
145
144
146
// NOTE: As a side effect, this saves the object.
145
147
$this->setLeaseDuration($retry);
146
148
147
149
$result = $this;
150
+
}
151
+
152
+
// NOTE: If this throws, we don't want it to cause the task to fail again,
153
+
// so execute it out here and just let the exception escape.
154
+
if ($did_succeed) {
155
+
foreach ($worker->getQueuedTasks() as $task) {
156
+
list($class, $data) = $task;
157
+
PhabricatorWorker::scheduleTask($class, $data);
158
+
}
148
159
}
149
160
150
161
return $result;