@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Allow tasks to yield to other tasks
Summary:
For Harbormaster tasks which want to poll or wait, this lets them say "try again a little later" without having to sleep and hold a queue slot.
This is basically the same as failing, except that we don't increment the failure counter. Instead, we just set the current lease to the correct length and then exit. The task will be retried after the lease expires.
Test Plan: Using both `bin/harbormaster` and `phd debug taskmaster`, ran a lot of waiting tasks through the queue, faking them to either yield or not yield in a controlled manner. The queue responded as expected, yielding tasks appropraitely and retrying them later.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: epriestley
Differential Revision: https://secure.phabricator.com/D8792
+73
-21
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
2239
2239
'PhabricatorWorkerTaskDetailController' => 'applications/daemon/controller/PhabricatorWorkerTaskDetailController.php',
2240
2240
'PhabricatorWorkerTaskUpdateController' => 'applications/daemon/controller/PhabricatorWorkerTaskUpdateController.php',
2241
2241
'PhabricatorWorkerTestCase' => 'infrastructure/daemon/workers/__tests__/PhabricatorWorkerTestCase.php',
2242
+
'PhabricatorWorkerYieldException' => 'infrastructure/daemon/workers/exception/PhabricatorWorkerYieldException.php',
2242
2243
'PhabricatorWorkingCopyDiscoveryTestCase' => 'applications/repository/engine/__tests__/PhabricatorWorkingCopyDiscoveryTestCase.php',
2243
2244
'PhabricatorWorkingCopyPullTestCase' => 'applications/repository/engine/__tests__/PhabricatorWorkingCopyPullTestCase.php',
2244
2245
'PhabricatorWorkingCopyTestCase' => 'applications/repository/engine/__tests__/PhabricatorWorkingCopyTestCase.php',
···
5126
5127
'PhabricatorWorkerTaskDetailController' => 'PhabricatorDaemonController',
5127
5128
'PhabricatorWorkerTaskUpdateController' => 'PhabricatorDaemonController',
5128
5129
'PhabricatorWorkerTestCase' => 'PhabricatorTestCase',
5130
+
'PhabricatorWorkerYieldException' => 'Exception',
5129
5131
'PhabricatorWorkingCopyDiscoveryTestCase' => 'PhabricatorWorkingCopyTestCase',
5130
5132
'PhabricatorWorkingCopyPullTestCase' => 'PhabricatorWorkingCopyTestCase',
5131
5133
'PhabricatorWorkingCopyTestCase' => 'PhabricatorTestCase',
+5
-5
src/applications/harbormaster/query/HarbormasterBuildQuery.php
+5
-5
src/applications/harbormaster/query/HarbormasterBuildQuery.php
···
132
132
private function buildWhereClause(AphrontDatabaseConnection $conn_r) {
133
133
$where = array();
134
134
135
-
if ($this->ids) {
135
+
if ($this->ids !== null) {
136
136
$where[] = qsprintf(
137
137
$conn_r,
138
138
'id IN (%Ld)',
139
139
$this->ids);
140
140
}
141
141
142
-
if ($this->phids) {
142
+
if ($this->phids !== null) {
143
143
$where[] = qsprintf(
144
144
$conn_r,
145
145
'phid in (%Ls)',
146
146
$this->phids);
147
147
}
148
148
149
-
if ($this->buildStatuses) {
149
+
if ($this->buildStatuses !== null) {
150
150
$where[] = qsprintf(
151
151
$conn_r,
152
152
'buildStatus in (%Ls)',
153
153
$this->buildStatuses);
154
154
}
155
155
156
-
if ($this->buildablePHIDs) {
156
+
if ($this->buildablePHIDs !== null) {
157
157
$where[] = qsprintf(
158
158
$conn_r,
159
159
'buildablePHID IN (%Ls)',
160
160
$this->buildablePHIDs);
161
161
}
162
162
163
-
if ($this->buildPlanPHIDs) {
163
+
if ($this->buildPlanPHIDs !== null) {
164
164
$where[] = qsprintf(
165
165
$conn_r,
166
166
'buildPlanPHID IN (%Ls)',
+13
-15
src/applications/harbormaster/step/HarbormasterWaitForPreviousBuildStepImplementation.php
+13
-15
src/applications/harbormaster/step/HarbormasterWaitForPreviousBuildStepImplementation.php
···
28
28
// finished.
29
29
$plan = $build->getBuildPlan();
30
30
31
-
$log = null;
32
-
$log_start = null;
33
-
$blockers = $this->getBlockers($object, $plan, $build);
34
-
while (count($blockers) > 0) {
35
-
if ($log === null) {
36
-
$log = $build->createLog($build_target, "waiting", "blockers");
37
-
$log_start = $log->start();
38
-
}
31
+
$log = $build->createLog($build_target, "waiting", "blockers");
32
+
$log_start = $log->start();
39
33
34
+
$blockers = $this->getBlockers($object, $plan, $build);
35
+
if ($blockers) {
40
36
$log->append("Blocked by: ".implode(",", $blockers)."\n");
37
+
}
38
+
$log->finalize($log_start);
41
39
42
-
// TODO: This should fail temporarily instead after setting the target to
43
-
// waiting, and thereby push the build into a waiting status.
44
-
sleep(1);
45
-
$blockers = $this->getBlockers($object, $plan, $build);
46
-
}
47
-
if ($log !== null) {
48
-
$log->finalize($log_start);
40
+
if ($blockers) {
41
+
throw new PhabricatorWorkerYieldException(15);
49
42
}
50
43
}
51
44
···
87
80
->execute();
88
81
$buildable_phids = mpull($buildables, 'getPHID');
89
82
83
+
if (!$buildable_phids) {
84
+
return array();
85
+
}
86
+
90
87
$builds = id(new HarbormasterBuildQuery())
91
88
->setViewer(PhabricatorUser::getOmnipotentUser())
92
89
->withBuildablePHIDs($buildable_phids)
···
101
98
102
99
return $blockers;
103
100
}
101
+
104
102
}
+4
src/applications/harbormaster/worker/HarbormasterTargetWorker.php
+4
src/applications/harbormaster/worker/HarbormasterTargetWorker.php
···
52
52
53
53
$target->setTargetStatus($next_status);
54
54
$target->save();
55
+
} catch (PhabricatorWorkerYieldException $ex) {
56
+
// If the target wants to yield, let that escape without further
57
+
// processing. We'll resume after the task retries.
58
+
throw $ex;
55
59
} catch (Exception $ex) {
56
60
phlog($ex);
57
61
+2
src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php
+2
src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php
···
21
21
if ($ex) {
22
22
if ($ex instanceof PhabricatorWorkerPermanentFailureException) {
23
23
$this->log("Task {$id} failed permanently.");
24
+
} else if ($ex instanceof PhabricatorWorkerYieldException) {
25
+
$this->log(pht('Task %s yielded.', $id));
24
26
} else {
25
27
$this->log("Task {$id} failed!");
26
28
throw new PhutilProxyException(
+15
-1
src/infrastructure/daemon/workers/PhabricatorWorker.php
+15
-1
src/infrastructure/daemon/workers/PhabricatorWorker.php
···
93
93
if (self::$runAllTasksInProcess) {
94
94
// Do the work in-process.
95
95
$worker = newv($task_class, array($data));
96
-
$worker->doWork();
96
+
97
+
while (true) {
98
+
try {
99
+
$worker->doWork();
100
+
break;
101
+
} catch (PhabricatorWorkerYieldException $ex) {
102
+
phlog(
103
+
pht(
104
+
'In-process task "%s" yielded for %s seconds, sleeping...',
105
+
$task_class,
106
+
$ex->getDuration()));
107
+
108
+
sleep($ex->getDuration());
109
+
}
110
+
}
97
111
98
112
// Now, save a task row and immediately archive it so we can return an
99
113
// object with a valid ID.
+22
src/infrastructure/daemon/workers/exception/PhabricatorWorkerYieldException.php
+22
src/infrastructure/daemon/workers/exception/PhabricatorWorkerYieldException.php
···
1
+
<?php
2
+
3
+
/**
4
+
* Allows tasks to yield to other tasks.
5
+
*
6
+
* If a worker throws this exception while processing a task, the task will be
7
+
* pushed toward the back of the queue and tried again later.
8
+
*/
9
+
final class PhabricatorWorkerYieldException extends Exception {
10
+
11
+
private $duration;
12
+
13
+
public function __construct($duration) {
14
+
$this->duration = $duration;
15
+
parent::__construct();
16
+
}
17
+
18
+
public function getDuration() {
19
+
return $this->duration;
20
+
}
21
+
22
+
}
+10
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
+10
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
···
133
133
PhabricatorWorkerArchiveTask::RESULT_FAILURE,
134
134
0);
135
135
$result->setExecutionException($ex);
136
+
} catch (PhabricatorWorkerYieldException $ex) {
137
+
$this->setExecutionException($ex);
138
+
139
+
$retry = $ex->getDuration();
140
+
$retry = max($retry, 5);
141
+
142
+
// NOTE: As a side effect, this saves the object.
143
+
$this->setLeaseDuration($retry);
144
+
145
+
$result = $this;
136
146
} catch (Exception $ex) {
137
147
$this->setExecutionException($ex);
138
148
$this->setFailureCount($this->getFailureCount() + 1);