@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Make Drydock reclaim unused resources when it reaches a resource limit
Summary:
Fixes T9994. Currently, when Drydock can't allocate a new resource because some limit has been reached, it waits patiently for a resource to become available.
It is possible that no resource will ever become available. Particularly with "Working Copy" resources, the new lease may want a copy of `rB`, but the resource may already be maxed out on `rA`.
Right now, no process exists to automatically reclaim the unused `rA`.
When we encounter this situation, try to reclaim one of the other resources if it is just sitting there unused.
Specifically:
- Add a "reclaim" command which means "release this resource //if// it is completely unused".
- Add a `bin/drydock reclaim` to send this command to every active resource.
- When we try to acquire a resource and can't, but only because of some kind of limit / utilization problem, try to release an unused resource to free up some room.
Test Plan:
- Set "Working Copy" resource limit to 1.
- Ran "Test Configuration" in `rA`, which worked.
- Ran "Test Configuration" in `rB`, which hung forever.
- Applied patch.
- Ran "Test Configuration" in `rB`, saw it reclaim the `rA` resource, use the slot, then succeed.
- Ran "Test Configuration" in `rA` again, saw it grab the slot back.
- Ran `bin/drydock reclaim` and saw it reclaim a bunch of old orphaned resources.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T9994
Differential Revision: https://secure.phabricator.com/D14819
+243
-2
+6
src/__phutil_library_map__.php
+6
src/__phutil_library_map__.php
···
883
883
'DrydockLeasePHIDType' => 'applications/drydock/phid/DrydockLeasePHIDType.php',
884
884
'DrydockLeaseQuery' => 'applications/drydock/query/DrydockLeaseQuery.php',
885
885
'DrydockLeaseQueuedLogType' => 'applications/drydock/logtype/DrydockLeaseQueuedLogType.php',
886
+
'DrydockLeaseReclaimLogType' => 'applications/drydock/logtype/DrydockLeaseReclaimLogType.php',
886
887
'DrydockLeaseReleaseController' => 'applications/drydock/controller/DrydockLeaseReleaseController.php',
887
888
'DrydockLeaseReleasedLogType' => 'applications/drydock/logtype/DrydockLeaseReleasedLogType.php',
888
889
'DrydockLeaseSearchEngine' => 'applications/drydock/query/DrydockLeaseSearchEngine.php',
···
900
901
'DrydockLogType' => 'applications/drydock/logtype/DrydockLogType.php',
901
902
'DrydockManagementCommandWorkflow' => 'applications/drydock/management/DrydockManagementCommandWorkflow.php',
902
903
'DrydockManagementLeaseWorkflow' => 'applications/drydock/management/DrydockManagementLeaseWorkflow.php',
904
+
'DrydockManagementReclaimWorkflow' => 'applications/drydock/management/DrydockManagementReclaimWorkflow.php',
903
905
'DrydockManagementReleaseLeaseWorkflow' => 'applications/drydock/management/DrydockManagementReleaseLeaseWorkflow.php',
904
906
'DrydockManagementReleaseResourceWorkflow' => 'applications/drydock/management/DrydockManagementReleaseResourceWorkflow.php',
905
907
'DrydockManagementUpdateLeaseWorkflow' => 'applications/drydock/management/DrydockManagementUpdateLeaseWorkflow.php',
···
926
928
'DrydockResourceListView' => 'applications/drydock/view/DrydockResourceListView.php',
927
929
'DrydockResourcePHIDType' => 'applications/drydock/phid/DrydockResourcePHIDType.php',
928
930
'DrydockResourceQuery' => 'applications/drydock/query/DrydockResourceQuery.php',
931
+
'DrydockResourceReclaimLogType' => 'applications/drydock/logtype/DrydockResourceReclaimLogType.php',
929
932
'DrydockResourceReleaseController' => 'applications/drydock/controller/DrydockResourceReleaseController.php',
930
933
'DrydockResourceSearchEngine' => 'applications/drydock/query/DrydockResourceSearchEngine.php',
931
934
'DrydockResourceStatus' => 'applications/drydock/constants/DrydockResourceStatus.php',
···
4819
4822
'DrydockLeasePHIDType' => 'PhabricatorPHIDType',
4820
4823
'DrydockLeaseQuery' => 'DrydockQuery',
4821
4824
'DrydockLeaseQueuedLogType' => 'DrydockLogType',
4825
+
'DrydockLeaseReclaimLogType' => 'DrydockLogType',
4822
4826
'DrydockLeaseReleaseController' => 'DrydockLeaseController',
4823
4827
'DrydockLeaseReleasedLogType' => 'DrydockLogType',
4824
4828
'DrydockLeaseSearchEngine' => 'PhabricatorApplicationSearchEngine',
···
4839
4843
'DrydockLogType' => 'Phobject',
4840
4844
'DrydockManagementCommandWorkflow' => 'DrydockManagementWorkflow',
4841
4845
'DrydockManagementLeaseWorkflow' => 'DrydockManagementWorkflow',
4846
+
'DrydockManagementReclaimWorkflow' => 'DrydockManagementWorkflow',
4842
4847
'DrydockManagementReleaseLeaseWorkflow' => 'DrydockManagementWorkflow',
4843
4848
'DrydockManagementReleaseResourceWorkflow' => 'DrydockManagementWorkflow',
4844
4849
'DrydockManagementUpdateLeaseWorkflow' => 'DrydockManagementWorkflow',
···
4871
4876
'DrydockResourceListView' => 'AphrontView',
4872
4877
'DrydockResourcePHIDType' => 'PhabricatorPHIDType',
4873
4878
'DrydockResourceQuery' => 'DrydockQuery',
4879
+
'DrydockResourceReclaimLogType' => 'DrydockLogType',
4874
4880
'DrydockResourceReleaseController' => 'DrydockResourceController',
4875
4881
'DrydockResourceSearchEngine' => 'PhabricatorApplicationSearchEngine',
4876
4882
'DrydockResourceStatus' => 'DrydockConstants',
+25
src/applications/drydock/logtype/DrydockLeaseReclaimLogType.php
+25
src/applications/drydock/logtype/DrydockLeaseReclaimLogType.php
···
1
+
<?php
2
+
3
+
final class DrydockLeaseReclaimLogType extends DrydockLogType {
4
+
5
+
const LOGCONST = 'core.lease.reclaim';
6
+
7
+
public function getLogTypeName() {
8
+
return pht('Reclaimed Resources');
9
+
}
10
+
11
+
public function getLogTypeIcon(array $data) {
12
+
return 'fa-refresh yellow';
13
+
}
14
+
15
+
public function renderLog(array $data) {
16
+
$viewer = $this->getViewer();
17
+
18
+
$resource_phids = idx($data, 'resourcePHIDs', array());
19
+
20
+
return pht(
21
+
'Reclaimed resource %s.',
22
+
$viewer->renderHandleList($resource_phids)->render());
23
+
}
24
+
25
+
}
+24
src/applications/drydock/logtype/DrydockResourceReclaimLogType.php
+24
src/applications/drydock/logtype/DrydockResourceReclaimLogType.php
···
1
+
<?php
2
+
3
+
final class DrydockResourceReclaimLogType extends DrydockLogType {
4
+
5
+
const LOGCONST = 'core.resource.reclaim';
6
+
7
+
public function getLogTypeName() {
8
+
return pht('Reclaimed');
9
+
}
10
+
11
+
public function getLogTypeIcon(array $data) {
12
+
return 'fa-refresh red';
13
+
}
14
+
15
+
public function renderLog(array $data) {
16
+
$viewer = $this->getViewer();
17
+
$reclaimer_phid = idx($data, 'reclaimerPHID');
18
+
19
+
return pht(
20
+
'Resource reclaimed by %s.',
21
+
$viewer->renderHandle($reclaimer_phid)->render());
22
+
}
23
+
24
+
}
+63
src/applications/drydock/management/DrydockManagementReclaimWorkflow.php
+63
src/applications/drydock/management/DrydockManagementReclaimWorkflow.php
···
1
+
<?php
2
+
3
+
final class DrydockManagementReclaimWorkflow
4
+
extends DrydockManagementWorkflow {
5
+
6
+
protected function didConstruct() {
7
+
$this
8
+
->setName('reclaim')
9
+
->setSynopsis(pht('Reclaim unused resources.'))
10
+
->setArguments(array());
11
+
}
12
+
13
+
public function execute(PhutilArgumentParser $args) {
14
+
$viewer = $this->getViewer();
15
+
$drydock_phid = id(new PhabricatorDrydockApplication())->getPHID();
16
+
17
+
PhabricatorWorker::setRunAllTasksInProcess(true);
18
+
19
+
$resources = id(new DrydockResourceQuery())
20
+
->setViewer($viewer)
21
+
->withStatuses(
22
+
array(
23
+
DrydockResourceStatus::STATUS_ACTIVE,
24
+
))
25
+
->execute();
26
+
foreach ($resources as $resource) {
27
+
$command = DrydockCommand::initializeNewCommand($viewer)
28
+
->setTargetPHID($resource->getPHID())
29
+
->setAuthorPHID($drydock_phid)
30
+
->setCommand(DrydockCommand::COMMAND_RECLAIM)
31
+
->save();
32
+
33
+
$resource->scheduleUpdate();
34
+
35
+
$resource = $resource->reload();
36
+
37
+
$name = pht(
38
+
'Resource %d: %s',
39
+
$resource->getID(),
40
+
$resource->getResourceName());
41
+
42
+
switch ($resource->getStatus()) {
43
+
case DrydockResourceStatus::STATUS_RELEASED:
44
+
case DrydockResourceStatus::STATUS_DESTROYED:
45
+
echo tsprintf(
46
+
"%s\n",
47
+
pht(
48
+
'Resource "%s" was reclaimed.',
49
+
$name));
50
+
break;
51
+
default:
52
+
echo tsprintf(
53
+
"%s\n",
54
+
pht(
55
+
'Resource "%s" could not be reclaimed.',
56
+
$name));
57
+
break;
58
+
}
59
+
}
60
+
61
+
}
62
+
63
+
}
+1
src/applications/drydock/storage/DrydockCommand.php
+1
src/applications/drydock/storage/DrydockCommand.php
+47
src/applications/drydock/worker/DrydockLeaseUpdateWorker.php
+47
src/applications/drydock/worker/DrydockLeaseUpdateWorker.php
···
179
179
// satisfy the lease, just not right now. This is a temporary failure,
180
180
// and we expect allocation to succeed eventually.
181
181
if (!$usable_blueprints) {
182
+
$blueprints = $this->rankBlueprints($blueprints, $lease);
183
+
184
+
// Try to actively reclaim unused resources. If we succeed, jump back
185
+
// into the queue in an effort to claim it.
186
+
foreach ($blueprints as $blueprint) {
187
+
$reclaimed = $this->reclaimResources($blueprint, $lease);
188
+
if ($reclaimed) {
189
+
$lease->logEvent(
190
+
DrydockLeaseReclaimLogType::LOGCONST,
191
+
array(
192
+
'resourcePHIDs' => array($reclaimed->getPHID()),
193
+
));
194
+
195
+
throw new PhabricatorWorkerYieldException(15);
196
+
}
197
+
}
198
+
182
199
$lease->logEvent(
183
200
DrydockLeaseWaitingForResourcesLogType::LOGCONST,
184
201
array(
···
439
456
assert_instances_of($blueprints, 'DrydockBlueprint');
440
457
441
458
$keep = array();
459
+
442
460
foreach ($blueprints as $key => $blueprint) {
443
461
if (!$blueprint->canAllocateResourceForLease($lease)) {
444
462
continue;
···
571
589
$resource_type,
572
590
$lease_type));
573
591
}
592
+
}
593
+
594
+
private function reclaimResources(
595
+
DrydockBlueprint $blueprint,
596
+
DrydockLease $lease) {
597
+
$viewer = $this->getViewer();
598
+
599
+
$resources = id(new DrydockResourceQuery())
600
+
->setViewer($viewer)
601
+
->withBlueprintPHIDs(array($blueprint->getPHID()))
602
+
->withStatuses(
603
+
array(
604
+
DrydockResourceStatus::STATUS_ACTIVE,
605
+
))
606
+
->execute();
607
+
608
+
// TODO: We could be much smarter about this and try to release long-unused
609
+
// resources, resources with many similar copies, old resources, resources
610
+
// that are cheap to rebuild, etc.
611
+
shuffle($resources);
612
+
613
+
foreach ($resources as $resource) {
614
+
if ($this->canReclaimResource($resource)) {
615
+
$this->reclaimResource($resource, $lease);
616
+
return $resource;
617
+
}
618
+
}
619
+
620
+
return null;
574
621
}
575
622
576
623
+21
-2
src/applications/drydock/worker/DrydockResourceUpdateWorker.php
+21
-2
src/applications/drydock/worker/DrydockResourceUpdateWorker.php
···
143
143
144
144
switch ($command->getCommand()) {
145
145
case DrydockCommand::COMMAND_RELEASE:
146
-
$this->releaseResource($resource);
146
+
$this->releaseResource($resource, null);
147
+
break;
148
+
case DrydockCommand::COMMAND_RECLAIM:
149
+
$reclaimer_phid = $command->getAuthorPHID();
150
+
$this->releaseResource($resource, $reclaimer_phid);
147
151
break;
148
152
}
149
153
}
···
188
192
/**
189
193
* @task release
190
194
*/
191
-
private function releaseResource(DrydockResource $resource) {
195
+
private function releaseResource(
196
+
DrydockResource $resource,
197
+
$reclaimer_phid) {
198
+
199
+
if ($reclaimer_phid) {
200
+
if (!$this->canReclaimResource($resource)) {
201
+
return;
202
+
}
203
+
204
+
$resource->logEvent(
205
+
DrydockResourceReclaimLogType::LOGCONST,
206
+
array(
207
+
'reclaimerPHID' => $reclaimer_phid,
208
+
));
209
+
}
210
+
192
211
$viewer = $this->getViewer();
193
212
$drydock_phid = id(new PhabricatorDrydockApplication())->getPHID();
194
213
+56
src/applications/drydock/worker/DrydockWorker.php
+56
src/applications/drydock/worker/DrydockWorker.php
···
195
195
return $this;
196
196
}
197
197
198
+
protected function canReclaimResource(DrydockResource $resource) {
199
+
$viewer = $this->getViewer();
200
+
201
+
// Don't reclaim a resource if it has been updated recently. If two
202
+
// leases are fighting, we don't want them to keep reclaming resources
203
+
// from one another forever without making progress, so make resources
204
+
// immune to reclamation for a little while after they activate or update.
205
+
206
+
// TODO: It would be nice to use a more narrow time here, like "last
207
+
// activation or lease release", but we don't currently store that
208
+
// anywhere.
209
+
210
+
$updated = $resource->getDateModified();
211
+
$now = PhabricatorTime::getNow();
212
+
$ago = ($now - $updated);
213
+
if ($ago < phutil_units('3 minutes in seconds')) {
214
+
return false;
215
+
}
216
+
217
+
$statuses = array(
218
+
DrydockLeaseStatus::STATUS_PENDING,
219
+
DrydockLeaseStatus::STATUS_ACQUIRED,
220
+
DrydockLeaseStatus::STATUS_ACTIVE,
221
+
DrydockLeaseStatus::STATUS_RELEASED,
222
+
DrydockLeaseStatus::STATUS_BROKEN,
223
+
);
224
+
225
+
// Don't reclaim resources that have any active leases.
226
+
$leases = id(new DrydockLeaseQuery())
227
+
->setViewer($viewer)
228
+
->withResourcePHIDs(array($resource->getPHID()))
229
+
->withStatuses($statuses)
230
+
->setLimit(1)
231
+
->execute();
232
+
if ($leases) {
233
+
return false;
234
+
}
235
+
236
+
return true;
237
+
}
238
+
239
+
protected function reclaimResource(
240
+
DrydockResource $resource,
241
+
DrydockLease $lease) {
242
+
$viewer = $this->getViewer();
243
+
244
+
$command = DrydockCommand::initializeNewCommand($viewer)
245
+
->setTargetPHID($resource->getPHID())
246
+
->setAuthorPHID($lease->getPHID())
247
+
->setCommand(DrydockCommand::COMMAND_RECLAIM)
248
+
->save();
249
+
250
+
$resource->scheduleUpdate();
251
+
252
+
return $this;
253
+
}
198
254
199
255
}