@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Split Harbormaster workers apart so build steps can run in parallel
Summary:
Ref T1049. Currently, the Harbormaster worker looks like this:
foreach (step) {
run_step(step);
}
This means steps can't ever be run in parallel. Instead, split it into two workers. The "Build" worker starts things off, and basically does:
update_build();
(We could theoretically do this in the original process because it should never take very long, but since there's a lock and it's a little bit complex it seemed cleaner to separate it.)
The "Target" worker runs an individual target (like a command, or an HTTP request, or whatever), then updates the build:
run_one_step(step);
update_build();
The new `update_build()` mechanism in `HarbormasterBuildEngine` does this, roughly:
figure_out_overall_status_of_all_steps();
if (build is done) { done(); }
if (build is fail) { fail(); }
foreach (step that is ready to run) {
queue_target_worker_for_step(step);
}
So, overall:
- The part of the code that updates Builds is completely separated from the part of the code that updates Targets.
- Targets can run in parallel.
Test Plan:
- Ran a bunch of builds via `bin/harbormaster build`.
- Ran a bunch of builds via web UI.
Reviewers: btrahan
Reviewed By: btrahan
CC: aran
Maniphest Tasks: T1049
Differential Revision: https://secure.phabricator.com/D7890
+306
-71
+7
-1
src/__phutil_library_map__.php
+7
-1
src/__phutil_library_map__.php
···
707
707
'HarbormasterBuildArtifact' => 'applications/harbormaster/storage/build/HarbormasterBuildArtifact.php',
708
708
'HarbormasterBuildArtifactQuery' => 'applications/harbormaster/query/HarbormasterBuildArtifactQuery.php',
709
709
'HarbormasterBuildCancelController' => 'applications/harbormaster/controller/HarbormasterBuildCancelController.php',
710
+
'HarbormasterBuildEngine' => 'applications/harbormaster/engine/HarbormasterBuildEngine.php',
710
711
'HarbormasterBuildItem' => 'applications/harbormaster/storage/build/HarbormasterBuildItem.php',
711
712
'HarbormasterBuildItemQuery' => 'applications/harbormaster/query/HarbormasterBuildItemQuery.php',
712
713
'HarbormasterBuildLog' => 'applications/harbormaster/storage/build/HarbormasterBuildLog.php',
···
757
758
'HarbormasterStepAddController' => 'applications/harbormaster/controller/HarbormasterStepAddController.php',
758
759
'HarbormasterStepDeleteController' => 'applications/harbormaster/controller/HarbormasterStepDeleteController.php',
759
760
'HarbormasterStepEditController' => 'applications/harbormaster/controller/HarbormasterStepEditController.php',
761
+
'HarbormasterTargetWorker' => 'applications/harbormaster/worker/HarbormasterTargetWorker.php',
760
762
'HarbormasterUIEventListener' => 'applications/harbormaster/event/HarbormasterUIEventListener.php',
763
+
'HarbormasterWorker' => 'applications/harbormaster/worker/HarbormasterWorker.php',
761
764
'HeraldAction' => 'applications/herald/storage/HeraldAction.php',
762
765
'HeraldAdapter' => 'applications/herald/adapter/HeraldAdapter.php',
763
766
'HeraldApplyTranscript' => 'applications/herald/storage/transcript/HeraldApplyTranscript.php',
···
3159
3162
),
3160
3163
'HarbormasterBuildArtifactQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
3161
3164
'HarbormasterBuildCancelController' => 'HarbormasterController',
3165
+
'HarbormasterBuildEngine' => 'Phobject',
3162
3166
'HarbormasterBuildItem' => 'HarbormasterDAO',
3163
3167
'HarbormasterBuildItemQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
3164
3168
'HarbormasterBuildLog' =>
···
3193
3197
),
3194
3198
'HarbormasterBuildTargetQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
3195
3199
'HarbormasterBuildViewController' => 'HarbormasterController',
3196
-
'HarbormasterBuildWorker' => 'PhabricatorWorker',
3200
+
'HarbormasterBuildWorker' => 'HarbormasterWorker',
3197
3201
'HarbormasterBuildable' =>
3198
3202
array(
3199
3203
0 => 'HarbormasterDAO',
···
3238
3242
'HarbormasterStepAddController' => 'HarbormasterController',
3239
3243
'HarbormasterStepDeleteController' => 'HarbormasterController',
3240
3244
'HarbormasterStepEditController' => 'HarbormasterController',
3245
+
'HarbormasterTargetWorker' => 'HarbormasterWorker',
3241
3246
'HarbormasterUIEventListener' => 'PhabricatorEventListener',
3247
+
'HarbormasterWorker' => 'PhabricatorWorker',
3242
3248
'HeraldAction' => 'HeraldDAO',
3243
3249
'HeraldApplyTranscript' => 'HeraldDAO',
3244
3250
'HeraldCapabilityManageGlobalRules' => 'PhabricatorPolicyCapability',
+223
src/applications/harbormaster/engine/HarbormasterBuildEngine.php
+223
src/applications/harbormaster/engine/HarbormasterBuildEngine.php
···
1
+
<?php
2
+
3
+
/**
4
+
* Moves a build forward by queuing build tasks, canceling or restarting the
5
+
* build, or failing it in response to task failures.
6
+
*/
7
+
final class HarbormasterBuildEngine extends Phobject {
8
+
9
+
private $build;
10
+
private $viewer;
11
+
private $newBuildTargets = array();
12
+
13
+
public function queueNewBuildTarget(HarbormasterBuildTarget $target) {
14
+
$this->newBuildTargets[] = $target;
15
+
return $this;
16
+
}
17
+
18
+
public function getNewBuildTargets() {
19
+
return $this->newBuildTargets;
20
+
}
21
+
22
+
public function setViewer(PhabricatorUser $viewer) {
23
+
$this->viewer = $viewer;
24
+
return $this;
25
+
}
26
+
27
+
public function getViewer() {
28
+
return $this->viewer;
29
+
}
30
+
31
+
public function setBuild(HarbormasterBuild $build) {
32
+
$this->build = $build;
33
+
return $this;
34
+
}
35
+
36
+
public function getBuild() {
37
+
return $this->build;
38
+
}
39
+
40
+
public function continueBuild() {
41
+
$build = $this->getBuild();
42
+
43
+
$lock_key = 'harbormaster.build:'.$build->getID();
44
+
$lock = PhabricatorGlobalLock::newLock($lock_key)->lock(15);
45
+
46
+
$build->reload();
47
+
48
+
try {
49
+
$this->updateBuild($build);
50
+
} catch (Exception $ex) {
51
+
// If any exception is raised, the build is marked as a failure and the
52
+
// exception is re-thrown (this ensures we don't leave builds in an
53
+
// inconsistent state).
54
+
$build->setBuildStatus(HarbormasterBuild::STATUS_ERROR);
55
+
$build->save();
56
+
57
+
$lock->unlock();
58
+
throw $ex;
59
+
}
60
+
61
+
$lock->unlock();
62
+
63
+
// NOTE: We queue new targets after releasing the lock so that in-process
64
+
// execution via `bin/harbormaster` does not reenter the locked region.
65
+
foreach ($this->getNewBuildTargets() as $target) {
66
+
$task = PhabricatorWorker::scheduleTask(
67
+
'HarbormasterTargetWorker',
68
+
array(
69
+
'targetID' => $target->getID(),
70
+
));
71
+
}
72
+
}
73
+
74
+
private function updateBuild(HarbormasterBuild $build) {
75
+
// TODO: Handle cancellation and restarts.
76
+
77
+
if ($build->getBuildStatus() == HarbormasterBuild::STATUS_PENDING) {
78
+
$this->destroyBuildTargets($build);
79
+
$build->setBuildStatus(HarbormasterBuild::STATUS_BUILDING);
80
+
$build->save();
81
+
}
82
+
83
+
if ($build->getBuildStatus() == HarbormasterBuild::STATUS_BUILDING) {
84
+
return $this->updateBuildSteps($build);
85
+
}
86
+
}
87
+
88
+
private function destroyBuildTargets(HarbormasterBuild $build) {
89
+
$targets = id(new HarbormasterBuildTargetQuery())
90
+
->setViewer($this->getViewer())
91
+
->withBuildPHIDs(array($build->getPHID()))
92
+
->execute();
93
+
foreach ($targets as $target) {
94
+
$target->delete();
95
+
}
96
+
}
97
+
98
+
private function updateBuildSteps(HarbormasterBuild $build) {
99
+
$targets = id(new HarbormasterBuildTargetQuery())
100
+
->setViewer($this->getViewer())
101
+
->withBuildPHIDs(array($build->getPHID()))
102
+
->execute();
103
+
$targets = mgroup($targets, 'getBuildStepPHID');
104
+
105
+
$steps = id(new HarbormasterBuildStepQuery())
106
+
->setViewer($this->getViewer())
107
+
->withBuildPlanPHIDs(array($build->getBuildPlan()->getPHID()))
108
+
->execute();
109
+
110
+
// Identify steps which are complete.
111
+
112
+
$complete = array();
113
+
$failed = array();
114
+
$waiting = array();
115
+
foreach ($steps as $step) {
116
+
$step_targets = idx($targets, $step->getPHID(), array());
117
+
118
+
if ($step_targets) {
119
+
$is_complete = true;
120
+
foreach ($step_targets as $target) {
121
+
// TODO: Move this to a top-level "status" field on BuildTarget.
122
+
if (!$target->getDetail('__done__')) {
123
+
$is_complete = false;
124
+
break;
125
+
}
126
+
}
127
+
128
+
$is_failed = false;
129
+
foreach ($step_targets as $target) {
130
+
// TODO: Move this to a top-level "status" field on BuildTarget.
131
+
if ($target->getDetail('__failed__')) {
132
+
$is_failed = true;
133
+
break;
134
+
}
135
+
}
136
+
137
+
$is_waiting = false;
138
+
} else {
139
+
$is_complete = false;
140
+
$is_failed = false;
141
+
$is_waiting = true;
142
+
}
143
+
144
+
if ($is_complete) {
145
+
$complete[$step->getPHID()] = true;
146
+
}
147
+
148
+
if ($is_failed) {
149
+
$failed[$step->getPHID()] = true;
150
+
}
151
+
152
+
if ($is_waiting) {
153
+
$waiting[$step->getPHID()] = true;
154
+
}
155
+
}
156
+
157
+
// If every step is complete, we're done with this build. Mark it passed
158
+
// and bail.
159
+
if (count($complete) == count($steps)) {
160
+
$build->setBuildStatus(HarbormasterBuild::STATUS_PASSED);
161
+
$build->save();
162
+
return;
163
+
}
164
+
165
+
// If any step failed, fail the whole build, then bail.
166
+
if (count($failed)) {
167
+
$build->setBuildStatus(HarbormasterBuild::STATUS_FAILED);
168
+
$build->save();
169
+
return;
170
+
}
171
+
172
+
// Identify all the steps which are ready to run (because all their
173
+
// depdendencies are complete).
174
+
175
+
$previous_step = null;
176
+
$runnable = array();
177
+
foreach ($steps as $step) {
178
+
// TODO: For now, we're hard coding sequential dependencies into build
179
+
// steps. In the future, we can be smart about this instead.
180
+
181
+
if ($previous_step) {
182
+
$dependencies = array($previous_step);
183
+
} else {
184
+
$dependencies = array();
185
+
}
186
+
187
+
if (isset($waiting[$step->getPHID()])) {
188
+
$can_run = true;
189
+
foreach ($dependencies as $dependency) {
190
+
if (empty($complete[$dependency->getPHID()])) {
191
+
$can_run = false;
192
+
break;
193
+
}
194
+
}
195
+
196
+
if ($can_run) {
197
+
$runnable[] = $step;
198
+
}
199
+
}
200
+
201
+
$previous_step = $step;
202
+
}
203
+
204
+
if (!$runnable) {
205
+
// TODO: This means the build is deadlocked, probably? It should not
206
+
// normally be possible, but we should communicate it more clearly.
207
+
$build->setBuildStatus(HarbormasterBuild::STATUS_FAILED);
208
+
$build->save();
209
+
return;
210
+
}
211
+
212
+
foreach ($runnable as $runnable_step) {
213
+
$target = HarbormasterBuildTarget::initializeNewBuildTarget(
214
+
$build,
215
+
$step,
216
+
$build->retrieveVariablesFromBuild());
217
+
$target->save();
218
+
219
+
$this->queueNewBuildTarget($target);
220
+
}
221
+
}
222
+
223
+
}
+8
-70
src/applications/harbormaster/worker/HarbormasterBuildWorker.php
+8
-70
src/applications/harbormaster/worker/HarbormasterBuildWorker.php
···
1
1
<?php
2
2
3
3
/**
4
-
* Run builds
4
+
* Start a build.
5
5
*/
6
-
final class HarbormasterBuildWorker extends PhabricatorWorker {
7
-
8
-
public function getRequiredLeaseTime() {
9
-
return 60 * 60 * 24;
10
-
}
6
+
final class HarbormasterBuildWorker extends HarbormasterWorker {
11
7
12
8
public function doWork() {
13
9
$data = $this->getTaskData();
14
10
$id = idx($data, 'buildID');
11
+
$viewer = $this->getViewer();
15
12
16
-
// Get a reference to the build.
17
13
$build = id(new HarbormasterBuildQuery())
18
-
->setViewer(PhabricatorUser::getOmnipotentUser())
19
-
->withBuildStatuses(array(HarbormasterBuild::STATUS_PENDING))
14
+
->setViewer($viewer)
20
15
->withIDs(array($id))
21
16
->executeOne();
22
17
if (!$build) {
···
24
19
pht('Invalid build ID "%s".', $id));
25
20
}
26
21
27
-
// It's possible for the user to request cancellation before
28
-
// a worker picks up a build. We check to see if the build
29
-
// is already cancelled, and return if it is.
30
-
if ($build->checkForCancellation()) {
31
-
return;
32
-
}
33
-
34
-
try {
35
-
$build->setBuildStatus(HarbormasterBuild::STATUS_BUILDING);
36
-
$build->save();
37
-
38
-
$buildable = $build->getBuildable();
39
-
$plan = $build->getBuildPlan();
40
-
41
-
$steps = $plan->loadOrderedBuildSteps();
42
-
43
-
// Perform the build.
44
-
foreach ($steps as $step) {
45
-
46
-
// Create the target at this step.
47
-
// TODO: Support variable artifacts.
48
-
$target = HarbormasterBuildTarget::initializeNewBuildTarget(
49
-
$build,
50
-
$step,
51
-
$build->retrieveVariablesFromBuild());
52
-
$target->save();
53
-
54
-
$implementation = $target->getImplementation();
55
-
if (!$implementation->validateSettings()) {
56
-
$build->setBuildStatus(HarbormasterBuild::STATUS_ERROR);
57
-
break;
58
-
}
59
-
$implementation->execute($build, $target);
60
-
if ($build->getBuildStatus() !== HarbormasterBuild::STATUS_BUILDING) {
61
-
break;
62
-
}
63
-
if ($build->checkForCancellation()) {
64
-
break;
65
-
}
66
-
}
67
-
68
-
// Check to see if the user requested cancellation. If they did and
69
-
// we get to here, they might have either cancelled too late, or the
70
-
// step isn't cancellation aware. In either case we ignore the result
71
-
// and move to a cancelled state.
72
-
$build->checkForCancellation();
73
-
74
-
// If we get to here, then the build has finished. Set it to passed
75
-
// if no build step explicitly set the status.
76
-
if ($build->getBuildStatus() === HarbormasterBuild::STATUS_BUILDING) {
77
-
$build->setBuildStatus(HarbormasterBuild::STATUS_PASSED);
78
-
}
79
-
$build->save();
80
-
} catch (Exception $e) {
81
-
// If any exception is raised, the build is marked as a failure and
82
-
// the exception is re-thrown (this ensures we don't leave builds
83
-
// in an inconsistent state).
84
-
$build->setBuildStatus(HarbormasterBuild::STATUS_ERROR);
85
-
$build->save();
86
-
throw $e;
87
-
}
22
+
id(new HarbormasterBuildEngine())
23
+
->setViewer($viewer)
24
+
->setBuild($build)
25
+
->continueBuild();
88
26
}
89
27
90
28
}
+59
src/applications/harbormaster/worker/HarbormasterTargetWorker.php
+59
src/applications/harbormaster/worker/HarbormasterTargetWorker.php
···
1
+
<?php
2
+
3
+
/**
4
+
* Execute a build target.
5
+
*/
6
+
final class HarbormasterTargetWorker extends HarbormasterWorker {
7
+
8
+
public function getRequiredLeaseTime() {
9
+
// This worker performs actual build work, which may involve a long wait
10
+
// on external systems.
11
+
return 60 * 60 * 24;
12
+
}
13
+
14
+
private function loadBuildTarget() {
15
+
$data = $this->getTaskData();
16
+
$id = idx($data, 'targetID');
17
+
18
+
$target = id(new HarbormasterBuildTargetQuery())
19
+
->withIDs(array($id))
20
+
->setViewer($this->getViewer())
21
+
->executeOne();
22
+
23
+
if (!$target) {
24
+
throw new PhabricatorWorkerPermanentFailureException(
25
+
pht(
26
+
'Bad build target ID "%d".',
27
+
$id));
28
+
}
29
+
30
+
return $target;
31
+
}
32
+
33
+
public function doWork() {
34
+
$target = $this->loadBuildTarget();
35
+
$build = $target->getBuild();
36
+
$viewer = $this->getViewer();
37
+
38
+
try {
39
+
$implementation = $target->getImplementation();
40
+
if (!$implementation->validateSettings()) {
41
+
$target->setDetail('__failed__', true);
42
+
$target->save();
43
+
} else {
44
+
$implementation->execute($build, $target);
45
+
$target->setDetail('__done__', true);
46
+
$target->save();
47
+
}
48
+
} catch (Exception $ex) {
49
+
$target->setDetail('__failed__', true);
50
+
$target->save();
51
+
}
52
+
53
+
id(new HarbormasterBuildEngine())
54
+
->setViewer($viewer)
55
+
->setBuild($build)
56
+
->continueBuild();
57
+
}
58
+
59
+
}