@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
When a Drydock Blueprint promises us a resource but can't deliver, continue believing in it
Summary:
Ref T13073. When a Blueprint says it will be able to allocate a resource but then throws an exception while attempting that allocation, we currently fail the lease permanently.
This is excessively harsh. This blueprint may have the best of intentions and have encountered a legitimately unforseeable failure (like a `vm.new` call to build a VM failed) and be able to succeed in the future.
Even if this blueprint is a dirty liar, other blueprints (or existing resources) may be able to satisfy the lease in the future.
Even if every blueprint is implemented incorrectly, leaving the lease alive lets it converge to success after the blueprints are fixed.
Instead of failing, log the issue and yield.
(In the future, it might make sense to distinguish more narrowly between "actually, all the resources are used up" and all other failure types, since the former is likely more routine and less concerning.)
Test Plan:
- Wrote a broken `Hoax` blueprint which always claims it can allocate but never actually allocates (just `throw` in `allocateResource()`).
- Used `bin/phd drydock lease` to acquire a Hoax lease.
- Before patch: lease abruptly failed permanently.
- After patch: lease yields after allocation fails.
{F5427747}
Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam
Maniphest Tasks: T13073
Differential Revision: https://secure.phabricator.com/D19070
+92
-2
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
1055
1055
'DrydockLeaseActivatedLogType' => 'applications/drydock/logtype/DrydockLeaseActivatedLogType.php',
1056
1056
'DrydockLeaseActivationFailureLogType' => 'applications/drydock/logtype/DrydockLeaseActivationFailureLogType.php',
1057
1057
'DrydockLeaseActivationYieldLogType' => 'applications/drydock/logtype/DrydockLeaseActivationYieldLogType.php',
1058
+
'DrydockLeaseAllocationFailureLogType' => 'applications/drydock/logtype/DrydockLeaseAllocationFailureLogType.php',
1058
1059
'DrydockLeaseController' => 'applications/drydock/controller/DrydockLeaseController.php',
1059
1060
'DrydockLeaseDatasource' => 'applications/drydock/typeahead/DrydockLeaseDatasource.php',
1060
1061
'DrydockLeaseDestroyedLogType' => 'applications/drydock/logtype/DrydockLeaseDestroyedLogType.php',
···
1106
1107
'DrydockResource' => 'applications/drydock/storage/DrydockResource.php',
1107
1108
'DrydockResourceActivationFailureLogType' => 'applications/drydock/logtype/DrydockResourceActivationFailureLogType.php',
1108
1109
'DrydockResourceActivationYieldLogType' => 'applications/drydock/logtype/DrydockResourceActivationYieldLogType.php',
1110
+
'DrydockResourceAllocationFailureLogType' => 'applications/drydock/logtype/DrydockResourceAllocationFailureLogType.php',
1109
1111
'DrydockResourceController' => 'applications/drydock/controller/DrydockResourceController.php',
1110
1112
'DrydockResourceDatasource' => 'applications/drydock/typeahead/DrydockResourceDatasource.php',
1111
1113
'DrydockResourceListController' => 'applications/drydock/controller/DrydockResourceListController.php',
···
6273
6275
'DrydockLeaseActivatedLogType' => 'DrydockLogType',
6274
6276
'DrydockLeaseActivationFailureLogType' => 'DrydockLogType',
6275
6277
'DrydockLeaseActivationYieldLogType' => 'DrydockLogType',
6278
+
'DrydockLeaseAllocationFailureLogType' => 'DrydockLogType',
6276
6279
'DrydockLeaseController' => 'DrydockController',
6277
6280
'DrydockLeaseDatasource' => 'PhabricatorTypeaheadDatasource',
6278
6281
'DrydockLeaseDestroyedLogType' => 'DrydockLogType',
···
6333
6336
),
6334
6337
'DrydockResourceActivationFailureLogType' => 'DrydockLogType',
6335
6338
'DrydockResourceActivationYieldLogType' => 'DrydockLogType',
6339
+
'DrydockResourceAllocationFailureLogType' => 'DrydockLogType',
6336
6340
'DrydockResourceController' => 'DrydockController',
6337
6341
'DrydockResourceDatasource' => 'PhabricatorTypeaheadDatasource',
6338
6342
'DrydockResourceListController' => 'DrydockResourceController',
+26
src/applications/drydock/logtype/DrydockLeaseAllocationFailureLogType.php
+26
src/applications/drydock/logtype/DrydockLeaseAllocationFailureLogType.php
···
1
+
<?php
2
+
3
+
final class DrydockLeaseAllocationFailureLogType extends DrydockLogType {
4
+
5
+
const LOGCONST = 'core.lease.allocation-failure';
6
+
7
+
public function getLogTypeName() {
8
+
return pht('Allocation Failed');
9
+
}
10
+
11
+
public function getLogTypeIcon(array $data) {
12
+
return 'fa-times red';
13
+
}
14
+
15
+
public function renderLog(array $data) {
16
+
$class = idx($data, 'class');
17
+
$message = idx($data, 'message');
18
+
19
+
return pht(
20
+
'One or more blueprints promised a new resource, but failed when '.
21
+
'allocating: [%s] %s',
22
+
$class,
23
+
$message);
24
+
}
25
+
26
+
}
+26
src/applications/drydock/logtype/DrydockResourceAllocationFailureLogType.php
+26
src/applications/drydock/logtype/DrydockResourceAllocationFailureLogType.php
···
1
+
<?php
2
+
3
+
final class DrydockResourceAllocationFailureLogType extends DrydockLogType {
4
+
5
+
const LOGCONST = 'core.resource.allocation-failure';
6
+
7
+
public function getLogTypeName() {
8
+
return pht('Allocation Failed');
9
+
}
10
+
11
+
public function getLogTypeIcon(array $data) {
12
+
return 'fa-times red';
13
+
}
14
+
15
+
public function renderLog(array $data) {
16
+
$class = idx($data, 'class');
17
+
$message = idx($data, 'message');
18
+
19
+
return pht(
20
+
'Blueprint failed to allocate a resource after claiming it would '.
21
+
'be able to: [%s] %s',
22
+
$class,
23
+
$message);
24
+
}
25
+
26
+
}
+36
-2
src/applications/drydock/worker/DrydockLeaseUpdateWorker.php
+36
-2
src/applications/drydock/worker/DrydockLeaseUpdateWorker.php
···
216
216
// this.
217
217
break;
218
218
} catch (Exception $ex) {
219
+
// This failure is not normally expected, so log it. It can be
220
+
// caused by something mundane and recoverable, however (see below
221
+
// for discussion).
222
+
223
+
// We log to the blueprint separately from the log to the lease:
224
+
// the lease is not attached to a blueprint yet so the lease log
225
+
// will not show up on the blueprint; more than one blueprint may
226
+
// fail; and the lease is not really impacted (and won't log) if at
227
+
// least one blueprint actually works.
228
+
229
+
$blueprint->logEvent(
230
+
DrydockResourceAllocationFailureLogType::LOGCONST,
231
+
array(
232
+
'class' => get_class($ex),
233
+
'message' => $ex->getMessage(),
234
+
));
235
+
219
236
$exceptions[] = $ex;
220
237
}
221
238
}
222
239
223
240
if (!$resources) {
224
-
throw new PhutilAggregateException(
241
+
// If one or more blueprints claimed that they would be able to
242
+
// allocate resources but none are actually able to allocate resources,
243
+
// log the failure and yield so we try again soon.
244
+
245
+
// This can happen if some unexpected issue occurs during allocation
246
+
// (for example, a call to build a VM fails for some reason) or if we
247
+
// raced another allocator and the blueprint is now full.
248
+
249
+
$ex = new PhutilAggregateException(
225
250
pht(
226
251
'All blueprints failed to allocate a suitable new resource when '.
227
-
'trying to allocate lease "%s".',
252
+
'trying to allocate lease ("%s").',
228
253
$lease->getPHID()),
229
254
$exceptions);
255
+
256
+
$lease->logEvent(
257
+
DrydockLeaseAllocationFailureLogType::LOGCONST,
258
+
array(
259
+
'class' => get_class($ex),
260
+
'message' => $ex->getMessage(),
261
+
));
262
+
263
+
throw new PhabricatorWorkerYieldException(15);
230
264
}
231
265
232
266
$resources = $this->removeUnacquirableResources($resources, $lease);