@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
In Harbormaster, release artifacts as soon as no waiting/running build steps will use them
Summary:
Ref T11153. If you have a build plan like this:
- Lease machine A.
- Lease machine B.
- Run client-tests on machine A.
- Run server-tests on machine B.
...and we get machine A quickly, then finish the tests, we currently do not release machine A until the whole plan finishes.
In the best case, this wastes resources (something else could be using that machine for a while).
In a worse case, this wastes a lot of resources (if machine B is slow to acquire, or the server tests are much slower than the client tests, machine A will get tied up for a really long time).
In the absolute worst case, this might deadlock things.
Instead, release artifacts as soon as no waiting/running steps take them as inputs. In this case, we'd release machine A as soon as we finished running the client tests.
In the case where machines A and B are resources of the same type, this should prevent deadlocks. In all cases, this should improve build throughput at least somewhat.
Test Plan:
I wrote this build plan which runs a "fast" step (10 seconds) and a "slow" step (120 seconds):
{F1691190}
Before the patch, running this build plan held the lease on the "fast" machine for the full 120 seconds, then released both leases at the same time at the very end.
After this patch, I ran this plan and observed the "fast" lease get released after 10 seconds, while the "slow" lease was held for the full 120.
(Also added some `var_dump()` into things to sanity check the logic; it appeared correct.)
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T11153
Differential Revision: https://secure.phabricator.com/D16145
+97
-11
+2
resources/sql/autopatches/20160617.harbormaster.01.arelease.sql
+2
resources/sql/autopatches/20160617.harbormaster.01.arelease.sql
+73
-4
src/applications/harbormaster/engine/HarbormasterBuildEngine.php
+73
-4
src/applications/harbormaster/engine/HarbormasterBuildEngine.php
···
9
9
private $build;
10
10
private $viewer;
11
11
private $newBuildTargets = array();
12
+
private $artifactReleaseQueue = array();
12
13
private $forceBuildableUpdate;
13
14
14
15
public function setForceBuildableUpdate($force_buildable_update) {
···
94
95
$this->updateBuildable($build->getBuildable());
95
96
}
96
97
98
+
$this->releaseQueuedArtifacts();
99
+
97
100
// If we are no longer building for any reason, release all artifacts.
98
101
if (!$build->isBuilding()) {
99
102
$this->releaseAllArtifacts($build);
···
149
152
}
150
153
151
154
private function updateBuildSteps(HarbormasterBuild $build) {
152
-
$targets = id(new HarbormasterBuildTargetQuery())
155
+
$all_targets = id(new HarbormasterBuildTargetQuery())
153
156
->setViewer($this->getViewer())
154
157
->withBuildPHIDs(array($build->getPHID()))
155
158
->withBuildGenerations(array($build->getBuildGeneration()))
156
159
->execute();
157
160
158
-
$this->updateWaitingTargets($targets);
161
+
$this->updateWaitingTargets($all_targets);
159
162
160
-
$targets = mgroup($targets, 'getBuildStepPHID');
163
+
$targets = mgroup($all_targets, 'getBuildStepPHID');
161
164
162
165
$steps = id(new HarbormasterBuildStepQuery())
163
166
->setViewer($this->getViewer())
164
167
->withBuildPlanPHIDs(array($build->getBuildPlan()->getPHID()))
165
168
->execute();
169
+
$steps = mpull($steps, null, 'getPHID');
166
170
167
171
// Identify steps which are in various states.
168
172
···
252
256
return;
253
257
}
254
258
259
+
// Release any artifacts which are not inputs to any remaining build
260
+
// step. We're done with these, so something else is free to use them.
261
+
$ongoing_phids = array_keys($queued + $waiting + $underway);
262
+
$ongoing_steps = array_select_keys($steps, $ongoing_phids);
263
+
$this->releaseUnusedArtifacts($all_targets, $ongoing_steps);
264
+
255
265
// Identify all the steps which are ready to run (because all their
256
266
// dependencies are complete).
257
267
···
295
305
296
306
297
307
/**
308
+
* Release any artifacts which aren't used by any running or waiting steps.
309
+
*
310
+
* This releases artifacts as soon as they're no longer used. This can be
311
+
* particularly relevant when a build uses multiple hosts since it returns
312
+
* hosts to the pool more quickly.
313
+
*
314
+
* @param list<HarbormasterBuildTarget> Targets in the build.
315
+
* @param list<HarbormasterBuildStep> List of running and waiting steps.
316
+
* @return void
317
+
*/
318
+
private function releaseUnusedArtifacts(array $targets, array $steps) {
319
+
assert_instances_of($targets, 'HarbormasterBuildTarget');
320
+
assert_instances_of($steps, 'HarbormasterBuildStep');
321
+
322
+
if (!$targets || !$steps) {
323
+
return;
324
+
}
325
+
326
+
$target_phids = mpull($targets, 'getPHID');
327
+
328
+
$artifacts = id(new HarbormasterBuildArtifactQuery())
329
+
->setViewer($this->getViewer())
330
+
->withBuildTargetPHIDs($target_phids)
331
+
->withIsReleased(false)
332
+
->execute();
333
+
if (!$artifacts) {
334
+
return;
335
+
}
336
+
337
+
// Collect all the artifacts that remaining build steps accept as inputs.
338
+
$must_keep = array();
339
+
foreach ($steps as $step) {
340
+
$inputs = $step->getStepImplementation()->getArtifactInputs();
341
+
foreach ($inputs as $input) {
342
+
$artifact_key = $input['key'];
343
+
$must_keep[$artifact_key] = true;
344
+
}
345
+
}
346
+
347
+
// Queue unreleased artifacts which no remaining step uses for immediate
348
+
// release.
349
+
foreach ($artifacts as $artifact) {
350
+
$key = $artifact->getArtifactKey();
351
+
if (isset($must_keep[$key])) {
352
+
continue;
353
+
}
354
+
355
+
$this->artifactReleaseQueue[] = $artifact;
356
+
}
357
+
}
358
+
359
+
360
+
/**
298
361
* Process messages which were sent to these targets, kicking applicable
299
362
* targets out of "Waiting" and into either "Passed" or "Failed".
300
363
*
···
488
551
$artifacts = id(new HarbormasterBuildArtifactQuery())
489
552
->setViewer(PhabricatorUser::getOmnipotentUser())
490
553
->withBuildTargetPHIDs($target_phids)
554
+
->withIsReleased(false)
491
555
->execute();
492
-
493
556
foreach ($artifacts as $artifact) {
494
557
$artifact->releaseArtifact();
495
558
}
559
+
}
496
560
561
+
private function releaseQueuedArtifacts() {
562
+
foreach ($this->artifactReleaseQueue as $key => $artifact) {
563
+
$artifact->releaseArtifact();
564
+
unset($this->artifactReleaseQueue[$key]);
565
+
}
497
566
}
498
567
499
568
}
+13
src/applications/harbormaster/query/HarbormasterBuildArtifactQuery.php
+13
src/applications/harbormaster/query/HarbormasterBuildArtifactQuery.php
···
9
9
private $artifactIndexes;
10
10
private $keyBuildPHID;
11
11
private $keyBuildGeneration;
12
+
private $isReleased;
12
13
13
14
public function withIDs(array $ids) {
14
15
$this->ids = $ids;
···
27
28
28
29
public function withArtifactIndexes(array $artifact_indexes) {
29
30
$this->artifactIndexes = $artifact_indexes;
31
+
return $this;
32
+
}
33
+
34
+
public function withIsReleased($released) {
35
+
$this->isReleased = $released;
30
36
return $this;
31
37
}
32
38
···
92
98
$conn,
93
99
'artifactIndex IN (%Ls)',
94
100
$this->artifactIndexes);
101
+
}
102
+
103
+
if ($this->isReleased !== null) {
104
+
$where[] = qsprintf(
105
+
$conn,
106
+
'isReleased = %d',
107
+
(int)$this->isReleased);
95
108
}
96
109
97
110
return $where;
-5
src/applications/harbormaster/step/HarbormasterBuildStepImplementation.php
-5
src/applications/harbormaster/step/HarbormasterBuildStepImplementation.php
···
103
103
/**
104
104
* Return the name of artifacts produced by this command.
105
105
*
106
-
* Something like:
107
-
*
108
-
* return array(
109
-
* 'some_name_input_by_user' => 'host');
110
-
*
111
106
* Future steps will calculate all available artifact mappings
112
107
* before them and filter on the type.
113
108
*
+9
-2
src/applications/harbormaster/storage/build/HarbormasterBuildArtifact.php
+9
-2
src/applications/harbormaster/storage/build/HarbormasterBuildArtifact.php
···
8
8
protected $artifactIndex;
9
9
protected $artifactKey;
10
10
protected $artifactData = array();
11
+
protected $isReleased = 0;
11
12
12
13
private $buildTarget = self::ATTACHABLE;
13
14
private $artifactImplementation;
···
29
30
'artifactType' => 'text32',
30
31
'artifactIndex' => 'bytes12',
31
32
'artifactKey' => 'text255',
33
+
'isReleased' => 'bool',
32
34
),
33
35
self::CONFIG_KEY_SCHEMA => array(
34
36
'key_artifact' => array(
···
83
85
}
84
86
85
87
public function releaseArtifact() {
86
-
$impl = $this->getArtifactImplementation();
88
+
if ($this->getIsReleased()) {
89
+
return $this;
90
+
}
87
91
92
+
$impl = $this->getArtifactImplementation();
88
93
if ($impl) {
89
94
$impl->releaseArtifact(PhabricatorUser::getOmnipotentUser());
90
95
}
91
96
92
-
return null;
97
+
return $this
98
+
->setIsReleased(1)
99
+
->save();
93
100
}
94
101
95
102
public function getArtifactImplementation() {