@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
In Harbormaster, make sure artifacts are destroyed even if a build is aborted
Summary:
Ref T9252. Currently, Harbormaster and Drydock work like this in some cases:
# Queue a lease for activation.
# Then, a little later, save the lease PHID somewhere.
# When the target/resource is destroyed, destroy the lease.
However, something can happen between (1) and (2). In Drydock this window is very short and the "something" would have to be a lighting strike or something similar, but in Harbormaster we wait until the resource activates to do (2) so the window can be many minutes long. In particular, a user can use "Abort Build" during those many minutes.
If they do, the target is destroyed but it doesn't yet have a record of the artifact, so the artifact isn't cleaned up.
Make these things work like this instead:
# Create a new lease and pre-generate a PHID for it.
# Save that PHID as something that needs to be cleaned up.
# Queue the lease for activation.
# When the target/resource is destroyed, destroy the lease if it exists.
This makes sure there's no step in the process where we might lose track of a lease/resource.
Also, clean up and standardize some other stuff I hit.
Test Plan:
- Stopped daemons.
- Restarted a build in Harbormaster.
- Stepped through the build one stage at a time using `bin/worker execute ...`.
- After the lease was queued, but before it activated, aborted the build.
- Processed the Harbormaster side of things only.
- Saw the lease get destroyed properly.
Reviewers: chad, hach-que
Reviewed By: hach-que
Maniphest Tasks: T9252
Differential Revision: https://secure.phabricator.com/D14234
+111
-55
+2
-1
src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php
+2
-1
src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php
···
262
262
array(
263
263
DrydockResourceStatus::STATUS_PENDING,
264
264
DrydockResourceStatus::STATUS_ACTIVE,
265
+
DrydockResourceStatus::STATUS_BROKEN,
265
266
DrydockResourceStatus::STATUS_RELEASED,
266
267
))
267
268
->execute();
268
269
269
270
$allocated_phids = array();
270
271
foreach ($pool as $resource) {
271
-
$allocated_phids[] = $resource->getAttribute('almanacDevicePHID');
272
+
$allocated_phids[] = $resource->getAttribute('almanacBindingPHID');
272
273
}
273
274
$allocated_phids = array_fuse($allocated_phids);
274
275
+1
-1
src/applications/drydock/blueprint/DrydockBlueprintImplementation.php
+1
-1
src/applications/drydock/blueprint/DrydockBlueprintImplementation.php
+14
-4
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
+14
-4
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
···
104
104
$host_lease = $this->newLease($blueprint)
105
105
->setResourceType('host')
106
106
->setOwnerPHID($resource_phid)
107
-
->setAttribute('workingcopy.resourcePHID', $resource_phid)
108
-
->queueForActivation();
107
+
->setAttribute('workingcopy.resourcePHID', $resource_phid);
108
+
109
+
$resource
110
+
->setAttribute('host.leasePHID', $host_lease->getPHID())
111
+
->save();
112
+
113
+
$host_lease->queueForActivation();
109
114
110
115
// TODO: Add some limits to the number of working copies we can have at
111
116
// once?
···
121
126
122
127
return $resource
123
128
->setAttribute('repositories.map', $map)
124
-
->setAttribute('host.leasePHID', $host_lease->getPHID())
125
129
->allocateResource();
126
130
}
127
131
···
165
169
DrydockBlueprint $blueprint,
166
170
DrydockResource $resource) {
167
171
168
-
$lease = $this->loadHostLease($resource);
172
+
try {
173
+
$lease = $this->loadHostLease($resource);
174
+
} catch (Exception $ex) {
175
+
// If we can't load the lease, assume we don't need to take any actions
176
+
// to destroy it.
177
+
return;
178
+
}
169
179
170
180
// Destroy the lease on the host.
171
181
$lease->releaseOnDestruction();
+1
-1
src/applications/drydock/logtype/DrydockLeaseWaitingForResourcesLogType.php
+1
-1
src/applications/drydock/logtype/DrydockLeaseWaitingForResourcesLogType.php
+33
-14
src/applications/drydock/storage/DrydockLease.php
+33
-14
src/applications/drydock/storage/DrydockLease.php
···
19
19
private $activateWhenAcquired = false;
20
20
private $slotLocks = array();
21
21
22
+
public static function initializeNewLease() {
23
+
$lease = new DrydockLease();
24
+
25
+
// Pregenerate a PHID so that the caller can set something up to release
26
+
// this lease before queueing it for activation.
27
+
$lease->setPHID($lease->generatePHID());
28
+
29
+
return $lease;
30
+
}
31
+
22
32
/**
23
33
* Flag this lease to be released when its destructor is called. This is
24
34
* mostly useful if you have a script which acquires, uses, and then releases
···
232
242
}
233
243
234
244
$this->openTransaction();
245
+
235
246
try {
236
247
DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks);
237
248
$this->slotLocks = array();
···
247
258
throw $ex;
248
259
}
249
260
250
-
try {
251
-
$this
252
-
->setResourcePHID($resource->getPHID())
253
-
->attachResource($resource)
254
-
->setStatus($new_status)
255
-
->save();
256
-
} catch (Exception $ex) {
257
-
$this->killTransaction();
258
-
throw $ex;
259
-
}
261
+
$this
262
+
->setResourcePHID($resource->getPHID())
263
+
->attachResource($resource)
264
+
->setStatus($new_status)
265
+
->save();
266
+
260
267
$this->saveTransaction();
261
268
262
269
$this->isAcquired = true;
···
295
302
296
303
$this->openTransaction();
297
304
298
-
$this
299
-
->setStatus(DrydockLeaseStatus::STATUS_ACTIVE)
300
-
->save();
301
-
305
+
try {
302
306
DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks);
303
307
$this->slotLocks = array();
308
+
} catch (DrydockSlotLockException $ex) {
309
+
$this->killTransaction();
310
+
311
+
$this->logEvent(
312
+
DrydockSlotLockFailureLogType::LOGCONST,
313
+
array(
314
+
'locks' => $ex->getLockMap(),
315
+
));
316
+
317
+
throw $ex;
318
+
}
319
+
320
+
$this
321
+
->setStatus(DrydockLeaseStatus::STATUS_ACTIVE)
322
+
->save();
304
323
305
324
$this->saveTransaction();
306
325
+27
-22
src/applications/drydock/storage/DrydockResource.php
+27
-22
src/applications/drydock/storage/DrydockResource.php
···
113
113
}
114
114
115
115
public function allocateResource() {
116
-
if ($this->getID()) {
117
-
throw new Exception(
118
-
pht(
119
-
'Trying to allocate a resource which has already been persisted. '.
120
-
'Only new resources may be allocated.'));
121
-
}
122
-
123
116
// We expect resources to have a pregenerated PHID, as they should have
124
117
// been created by a call to DrydockBlueprint->newResourceTemplate().
125
118
if (!$this->getPHID()) {
···
155
148
} catch (DrydockSlotLockException $ex) {
156
149
$this->killTransaction();
157
150
158
-
// NOTE: We have to log this on the blueprint, as the resource is not
159
-
// going to be saved so the PHID will vanish.
160
-
$this->getBlueprint()->logEvent(
151
+
if ($this->getID()) {
152
+
$log_target = $this;
153
+
} else {
154
+
// If we don't have an ID, we have to log this on the blueprint, as the
155
+
// resource is not going to be saved so the PHID will vanish.
156
+
$log_target = $this->getBlueprint();
157
+
}
158
+
$log_target->logEvent(
161
159
DrydockSlotLockFailureLogType::LOGCONST,
162
160
array(
163
161
'locks' => $ex->getLockMap(),
···
166
164
throw $ex;
167
165
}
168
166
169
-
try {
170
-
$this
171
-
->setStatus($new_status)
172
-
->save();
173
-
} catch (Exception $ex) {
174
-
$this->killTransaction();
175
-
throw $ex;
176
-
}
167
+
$this
168
+
->setStatus($new_status)
169
+
->save();
177
170
178
171
$this->saveTransaction();
179
172
···
210
203
211
204
$this->openTransaction();
212
205
213
-
$this
214
-
->setStatus(DrydockResourceStatus::STATUS_ACTIVE)
215
-
->save();
216
-
206
+
try {
217
207
DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks);
218
208
$this->slotLocks = array();
209
+
} catch (DrydockSlotLockException $ex) {
210
+
$this->killTransaction();
211
+
212
+
$this->logEvent(
213
+
DrydockSlotLockFailureLogType::LOGCONST,
214
+
array(
215
+
'locks' => $ex->getLockMap(),
216
+
));
217
+
218
+
throw $ex;
219
+
}
220
+
221
+
$this
222
+
->setStatus(DrydockResourceStatus::STATUS_ACTIVE)
223
+
->save();
219
224
220
225
$this->saveTransaction();
221
226
+1
src/applications/drydock/storage/DrydockSlotLock.php
+1
src/applications/drydock/storage/DrydockSlotLock.php
+12
-2
src/applications/harbormaster/artifact/HarbormasterDrydockLeaseArtifact.php
+12
-2
src/applications/harbormaster/artifact/HarbormasterDrydockLeaseArtifact.php
···
29
29
}
30
30
31
31
public function willCreateArtifact(PhabricatorUser $actor) {
32
-
$this->loadArtifactLease($actor);
32
+
// We don't load the lease here because it's expected that artifacts are
33
+
// created before leases actually exist. This guarantees that the leases
34
+
// will be cleaned up.
33
35
}
34
36
35
37
public function loadArtifactLease(PhabricatorUser $viewer) {
···
51
53
}
52
54
53
55
public function releaseArtifact(PhabricatorUser $actor) {
54
-
$lease = $this->loadArtifactLease($actor);
56
+
try {
57
+
$lease = $this->loadArtifactLease($actor);
58
+
} catch (Exception $ex) {
59
+
// If we can't load the lease, treat it as already released. Artifacts
60
+
// are generated before leases are queued, so it's possible to arrive
61
+
// here under normal conditions.
62
+
return;
63
+
}
64
+
55
65
if (!$lease->canRelease()) {
56
66
return;
57
67
}
+13
-9
src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php
+13
-9
src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php
···
41
41
$working_copy_type = id(new DrydockWorkingCopyBlueprintImplementation())
42
42
->getType();
43
43
44
-
$lease = id(new DrydockLease())
44
+
$lease = DrydockLease::initializeNewLease()
45
45
->setResourceType($working_copy_type)
46
46
->setOwnerPHID($build_target->getPHID());
47
47
···
54
54
$lease->setAwakenTaskIDs(array($task_id));
55
55
}
56
56
57
+
// TODO: Maybe add a method to mark artifacts like this as pending?
58
+
59
+
// Create the artifact now so that the lease is always disposed of, even
60
+
// if this target is aborted.
61
+
$build_target->createArtifact(
62
+
$viewer,
63
+
$settings['name'],
64
+
HarbormasterWorkingCopyArtifact::ARTIFACTCONST,
65
+
array(
66
+
'drydockLeasePHID' => $lease->getPHID(),
67
+
));
68
+
57
69
$lease->queueForActivation();
58
70
59
71
$build_target
···
73
85
'Lease "%s" never activated.',
74
86
$lease->getPHID()));
75
87
}
76
-
77
-
$artifact = $build_target->createArtifact(
78
-
$viewer,
79
-
$settings['name'],
80
-
HarbormasterWorkingCopyArtifact::ARTIFACTCONST,
81
-
array(
82
-
'drydockLeasePHID' => $lease->getPHID(),
83
-
));
84
88
}
85
89
86
90
public function getArtifactOutputs() {
+2
-1
src/infrastructure/daemon/workers/management/PhabricatorWorkerManagementExecuteWorkflow.php
+2
-1
src/infrastructure/daemon/workers/management/PhabricatorWorkerManagementExecuteWorkflow.php
+5
src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php
+5
src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php
···
1398
1398
'Setting retention policy for "%s" to %s days.',
1399
1399
),
1400
1400
1401
+
'Waiting %s second(s) for lease to activate.' => array(
1402
+
'Waiting a second for lease to activate.',
1403
+
'Waiting %s seconds for lease to activate.',
1404
+
),
1405
+
1401
1406
);
1402
1407
}
1403
1408