@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Improve error and exception handling for Drydock resources
Summary:
Ref T9252. Currently, error handling behavior isn't great and a lot of errors aren't dealt with properly. Try to improve this by making default behaviors better:
- Yields, slot lock exceptions, and aggregate or proxy exceptions containing an excpetion of these types turn into yields.
- All other exceptions are considered permanent failures. They break the resource and
This feels a little bit "magical" but I want to try to get the default behaviors to align reasonably well with expectations so that blueprints mostly don't need to have a ton of error handling. This will probably need at least some refinement down the road, but it's a reasonable rule for all exception/error conditions we currently have.
Test Plan: I did a clean build, but haven't vetted this super thoroughly. Next diff will do the same thing to leases, then I'll work on stabilizing this code better.
Reviewers: chad, hach-que
Reviewed By: hach-que
Maniphest Tasks: T9252
Differential Revision: https://secure.phabricator.com/D14211
+152
-16
-1
src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php
-1
src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php
+11
-6
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
+11
-6
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
···
297
297
298
298
foreach ($phids as $phid) {
299
299
if (empty($repositories[$phid])) {
300
-
// TODO: Permanent failure.
301
300
throw new Exception(
302
301
pht(
303
302
'Repository PHID "%s" does not exist.',
···
306
305
}
307
306
308
307
foreach ($repositories as $repository) {
309
-
switch ($repository->getVersionControlSystem()) {
308
+
$repository_vcs = $repository->getVersionControlSystem();
309
+
switch ($repository_vcs) {
310
310
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
311
311
break;
312
312
default:
313
-
// TODO: Permanent failure.
314
-
throw new Exception(pht('Unsupported VCS!'));
313
+
throw new Exception(
314
+
pht(
315
+
'Repository ("%s") has unsupported VCS ("%s").',
316
+
$repository->getPHID(),
317
+
$repository_vcs));
315
318
}
316
319
}
317
320
···
328
331
->withPHIDs(array($lease_phid))
329
332
->executeOne();
330
333
if (!$lease) {
331
-
// TODO: Permanent failure.
332
-
throw new Exception(pht('Unable to load lease "%s".', $lease_phid));
334
+
throw new Exception(
335
+
pht(
336
+
'Unable to load lease ("%s").',
337
+
$lease_phid));
333
338
}
334
339
335
340
return $lease;
+100
-9
src/applications/drydock/worker/DrydockResourceUpdateWorker.php
+100
-9
src/applications/drydock/worker/DrydockResourceUpdateWorker.php
···
1
1
<?php
2
2
3
3
/**
4
+
* @task update Updating Resources
4
5
* @task command Processing Commands
5
6
* @task activate Activating Resources
6
7
* @task release Releasing Resources
8
+
* @task break Breaking Resources
7
9
* @task destroy Destroying Resources
8
10
*/
9
11
final class DrydockResourceUpdateWorker extends DrydockWorker {
···
19
21
20
22
try {
21
23
$resource = $this->loadResource($resource_phid);
22
-
$this->updateResource($resource);
24
+
$this->handleUpdate($resource);
23
25
} catch (Exception $ex) {
24
26
$lock->unlock();
25
27
throw $ex;
···
28
30
$lock->unlock();
29
31
}
30
32
33
+
34
+
/* -( Updating Resources )------------------------------------------------- */
35
+
36
+
37
+
/**
38
+
* Update a resource, handling exceptions thrown during the update.
39
+
*
40
+
* @param DrydockReosource Resource to update.
41
+
* @return void
42
+
* @task update
43
+
*/
44
+
private function handleUpdate(DrydockResource $resource) {
45
+
try {
46
+
$this->updateResource($resource);
47
+
} catch (Exception $ex) {
48
+
if ($this->isTemporaryException($ex)) {
49
+
$this->yieldResource($resource, $ex);
50
+
} else {
51
+
$this->breakResource($resource, $ex);
52
+
}
53
+
}
54
+
}
55
+
56
+
57
+
/**
58
+
* Update a resource.
59
+
*
60
+
* @param DrydockResource Resource to update.
61
+
* @return void
62
+
* @task update
63
+
*/
31
64
private function updateResource(DrydockResource $resource) {
32
65
$this->processResourceCommands($resource);
33
66
···
49
82
}
50
83
51
84
$this->yieldIfExpiringResource($resource);
85
+
}
86
+
87
+
88
+
/**
89
+
* Convert a temporary exception into a yield.
90
+
*
91
+
* @param DrydockResource Resource to yield.
92
+
* @param Exception Temporary exception worker encountered.
93
+
* @task update
94
+
*/
95
+
private function yieldResource(DrydockResource $resource, Exception $ex) {
96
+
$duration = $this->getYieldDurationFromException($ex);
97
+
98
+
$resource->logEvent(
99
+
DrydockResourceActivationYieldLogType::LOGCONST,
100
+
array(
101
+
'duration' => $duration,
102
+
));
103
+
104
+
throw new PhabricatorWorkerYieldException($duration);
52
105
}
53
106
54
107
···
138
191
$viewer = $this->getViewer();
139
192
$drydock_phid = id(new PhabricatorDrydockApplication())->getPHID();
140
193
141
-
$resource->openTransaction();
142
-
$resource
143
-
->setStatus(DrydockResourceStatus::STATUS_RELEASED)
144
-
->save();
145
-
146
-
// TODO: Hold slot locks until destruction?
147
-
DrydockSlotLock::releaseLocks($resource->getPHID());
148
-
$resource->saveTransaction();
194
+
$resource
195
+
->setStatus(DrydockResourceStatus::STATUS_RELEASED)
196
+
->save();
149
197
150
198
$statuses = array(
151
199
DrydockLeaseStatus::STATUS_PENDING,
···
173
221
}
174
222
175
223
224
+
/* -( Breaking Resources )------------------------------------------------- */
225
+
226
+
227
+
/**
228
+
* @task break
229
+
*/
230
+
private function breakResource(DrydockResource $resource, Exception $ex) {
231
+
switch ($resource->getStatus()) {
232
+
case DrydockResourceStatus::STATUS_BROKEN:
233
+
case DrydockResourceStatus::STATUS_RELEASED:
234
+
case DrydockResourceStatus::STATUS_DESTROYED:
235
+
// If the resource was already broken, just throw a normal exception.
236
+
// This will retry the task eventually.
237
+
throw new PhutilProxyException(
238
+
pht(
239
+
'Unexpected failure while destroying resource ("%s").',
240
+
$resource->getPHID()),
241
+
$ex);
242
+
}
243
+
244
+
$resource
245
+
->setStatus(DrydockResourceStatus::STATUS_BROKEN)
246
+
->save();
247
+
248
+
$resource->scheduleUpdate();
249
+
250
+
$resource->logEvent(
251
+
DrydockResourceActivationFailureLogType::LOGCONST,
252
+
array(
253
+
'class' => get_class($ex),
254
+
'message' => $ex->getMessage(),
255
+
));
256
+
257
+
throw new PhabricatorWorkerPermanentFailureException(
258
+
pht(
259
+
'Permanent failure while activating resource ("%s"): %s',
260
+
$resource->getPHID(),
261
+
$ex->getMessage()));
262
+
}
263
+
264
+
176
265
/* -( Destroying Resources )----------------------------------------------- */
177
266
178
267
···
182
271
private function destroyResource(DrydockResource $resource) {
183
272
$blueprint = $resource->getBlueprint();
184
273
$blueprint->destroyResource($resource);
274
+
275
+
DrydockSlotLock::releaseLocks($resource->getPHID());
185
276
186
277
$resource
187
278
->setStatus(DrydockResourceStatus::STATUS_DESTROYED)
+41
src/applications/drydock/worker/DrydockWorker.php
+41
src/applications/drydock/worker/DrydockWorker.php
···
114
114
throw new PhabricatorWorkerYieldException($expires - $now);
115
115
}
116
116
117
+
protected function isTemporaryException(Exception $ex) {
118
+
if ($ex instanceof PhabricatorWorkerYieldException) {
119
+
return true;
120
+
}
121
+
122
+
if ($ex instanceof DrydockSlotLockException) {
123
+
return true;
124
+
}
125
+
126
+
if ($ex instanceof PhutilAggregateException) {
127
+
$any_temporary = false;
128
+
foreach ($ex->getExceptions() as $sub) {
129
+
if ($this->isTemporaryException($sub)) {
130
+
$any_temporary = true;
131
+
break;
132
+
}
133
+
}
134
+
if ($any_temporary) {
135
+
return true;
136
+
}
137
+
}
138
+
139
+
if ($ex instanceof PhutilProxyException) {
140
+
return $this->isTemporaryException($ex->getPreviousException());
141
+
}
142
+
143
+
return false;
144
+
}
145
+
146
+
protected function getYieldDurationFromException(Exception $ex) {
147
+
if ($ex instanceof PhabricatorWorkerYieldException) {
148
+
return $ex->getDuration();
149
+
}
150
+
151
+
if ($ex instanceof DrydockSlotLockException) {
152
+
return 5;
153
+
}
154
+
155
+
return 15;
156
+
}
157
+
117
158
}