@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
When a lease acquires a resource but the resource fails to activate, throw the lease back in the pool
Summary:
Depends on D19075. Ref T13073. If a lease acquires a resource but finds that the resource builds directly into a dead state (which can happen for any number of reasonable reasons), reset the lease and throw it back in the pool.
This isn't the lease's fault and it hasn't caused any side effects or done anything we can't undo, so we can safely reset it and throw it back in the pool.
Test Plan:
- Created a blueprint which throws from `allocateResource()` so that resources never activate.
- Tried to lease it with `bin/drydock lease ...`.
- Before patch: lease was broken and destroyed after a failed activation.
- After patch: lease was returned to the pool after a failed activation.
Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam
Maniphest Tasks: T13073
Differential Revision: https://secure.phabricator.com/D19076
+59
-2
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
1000
1000
'DoorkeeperSchemaSpec' => 'applications/doorkeeper/storage/DoorkeeperSchemaSpec.php',
1001
1001
'DoorkeeperTagView' => 'applications/doorkeeper/view/DoorkeeperTagView.php',
1002
1002
'DoorkeeperTagsController' => 'applications/doorkeeper/controller/DoorkeeperTagsController.php',
1003
+
'DrydockAcquiredBrokenResourceException' => 'applications/drydock/exception/DrydockAcquiredBrokenResourceException.php',
1003
1004
'DrydockAlmanacServiceHostBlueprintImplementation' => 'applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php',
1004
1005
'DrydockApacheWebrootInterface' => 'applications/drydock/interface/webroot/DrydockApacheWebrootInterface.php',
1005
1006
'DrydockAuthorization' => 'applications/drydock/storage/DrydockAuthorization.php',
···
1065
1066
'DrydockLeasePHIDType' => 'applications/drydock/phid/DrydockLeasePHIDType.php',
1066
1067
'DrydockLeaseQuery' => 'applications/drydock/query/DrydockLeaseQuery.php',
1067
1068
'DrydockLeaseQueuedLogType' => 'applications/drydock/logtype/DrydockLeaseQueuedLogType.php',
1069
+
'DrydockLeaseReacquireLogType' => 'applications/drydock/logtype/DrydockLeaseReacquireLogType.php',
1068
1070
'DrydockLeaseReclaimLogType' => 'applications/drydock/logtype/DrydockLeaseReclaimLogType.php',
1069
1071
'DrydockLeaseReleaseController' => 'applications/drydock/controller/DrydockLeaseReleaseController.php',
1070
1072
'DrydockLeaseReleasedLogType' => 'applications/drydock/logtype/DrydockLeaseReleasedLogType.php',
···
6199
6201
'DoorkeeperSchemaSpec' => 'PhabricatorConfigSchemaSpec',
6200
6202
'DoorkeeperTagView' => 'AphrontView',
6201
6203
'DoorkeeperTagsController' => 'PhabricatorController',
6204
+
'DrydockAcquiredBrokenResourceException' => 'Exception',
6202
6205
'DrydockAlmanacServiceHostBlueprintImplementation' => 'DrydockBlueprintImplementation',
6203
6206
'DrydockApacheWebrootInterface' => 'DrydockWebrootInterface',
6204
6207
'DrydockAuthorization' => array(
···
6285
6288
'DrydockLeasePHIDType' => 'PhabricatorPHIDType',
6286
6289
'DrydockLeaseQuery' => 'DrydockQuery',
6287
6290
'DrydockLeaseQueuedLogType' => 'DrydockLogType',
6291
+
'DrydockLeaseReacquireLogType' => 'DrydockLogType',
6288
6292
'DrydockLeaseReclaimLogType' => 'DrydockLogType',
6289
6293
'DrydockLeaseReleaseController' => 'DrydockLeaseController',
6290
6294
'DrydockLeaseReleasedLogType' => 'DrydockLogType',
+4
src/applications/drydock/exception/DrydockAcquiredBrokenResourceException.php
+4
src/applications/drydock/exception/DrydockAcquiredBrokenResourceException.php
+26
src/applications/drydock/logtype/DrydockLeaseReacquireLogType.php
+26
src/applications/drydock/logtype/DrydockLeaseReacquireLogType.php
···
1
+
<?php
2
+
3
+
final class DrydockLeaseReacquireLogType extends DrydockLogType {
4
+
5
+
const LOGCONST = 'core.lease.reacquire';
6
+
7
+
public function getLogTypeName() {
8
+
return pht('Reacquiring Resource');
9
+
}
10
+
11
+
public function getLogTypeIcon(array $data) {
12
+
return 'fa-refresh yellow';
13
+
}
14
+
15
+
public function renderLog(array $data) {
16
+
$class = idx($data, 'class');
17
+
$message = idx($data, 'message');
18
+
19
+
return pht(
20
+
'Lease acquired a resource but failed to activate; acquisition '.
21
+
'will be retried: [%s] %s',
22
+
$class,
23
+
$message);
24
+
}
25
+
26
+
}
+25
-2
src/applications/drydock/worker/DrydockLeaseUpdateWorker.php
+25
-2
src/applications/drydock/worker/DrydockLeaseUpdateWorker.php
···
43
43
private function handleUpdate(DrydockLease $lease) {
44
44
try {
45
45
$this->updateLease($lease);
46
+
} catch (DrydockAcquiredBrokenResourceException $ex) {
47
+
// If this lease acquired a resource but failed to activate, we don't
48
+
// need to break the lease. We can throw it back in the pool and let
49
+
// it take another shot at acquiring a new resource.
50
+
51
+
// Before we throw it back, release any locks the lease is holding.
52
+
DrydockSlotLock::releaseLocks($lease->getPHID());
53
+
54
+
$lease
55
+
->setStatus(DrydockLeaseStatus::STATUS_PENDING)
56
+
->save();
57
+
58
+
$lease->logEvent(
59
+
DrydockLeaseReacquireLogType::LOGCONST,
60
+
array(
61
+
'class' => get_class($ex),
62
+
'message' => $ex->getMessage(),
63
+
));
64
+
65
+
$this->yieldLease($lease, $ex);
46
66
} catch (Exception $ex) {
47
67
if ($this->isTemporaryException($ex)) {
48
68
$this->yieldLease($lease, $ex);
···
749
769
}
750
770
751
771
if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) {
752
-
throw new Exception(
772
+
throw new DrydockAcquiredBrokenResourceException(
753
773
pht(
754
-
'Trying to activate lease on a dead resource (in status "%s").',
774
+
'Trying to activate lease ("%s") on a resource ("%s") in '.
775
+
'the wrong status ("%s").',
776
+
$lease->getPHID(),
777
+
$resource->getPHID(),
755
778
$resource_status));
756
779
}
757
780