@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Give Drydock resources and leases a real "destroy" lifecycle phase
Summary: Ref T9252. Some leases or resources may need to remove data, tear down VMs, etc., during cleanup. After they are released, queue a "destroy" phase for performing teardown.
Test Plan:
- Used `bin/drydock lease ...` to create a working copy lease.
- Used `bin/drydock release-lease` and `bin/drydock release-resource` to release the lease and then the working copy and host.
- Saw working copy and host get destroyed and cleaned up properly.
Reviewers: hach-que, chad
Reviewed By: chad
Maniphest Tasks: T6569, T9252
Differential Revision: https://secure.phabricator.com/D14144
+264
-36
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
832
832
'DrydockLease' => 'applications/drydock/storage/DrydockLease.php',
833
833
'DrydockLeaseController' => 'applications/drydock/controller/DrydockLeaseController.php',
834
834
'DrydockLeaseDatasource' => 'applications/drydock/typeahead/DrydockLeaseDatasource.php',
835
+
'DrydockLeaseDestroyWorker' => 'applications/drydock/worker/DrydockLeaseDestroyWorker.php',
835
836
'DrydockLeaseListController' => 'applications/drydock/controller/DrydockLeaseListController.php',
836
837
'DrydockLeaseListView' => 'applications/drydock/view/DrydockLeaseListView.php',
837
838
'DrydockLeasePHIDType' => 'applications/drydock/phid/DrydockLeasePHIDType.php',
···
859
860
'DrydockResource' => 'applications/drydock/storage/DrydockResource.php',
860
861
'DrydockResourceController' => 'applications/drydock/controller/DrydockResourceController.php',
861
862
'DrydockResourceDatasource' => 'applications/drydock/typeahead/DrydockResourceDatasource.php',
863
+
'DrydockResourceDestroyWorker' => 'applications/drydock/worker/DrydockResourceDestroyWorker.php',
862
864
'DrydockResourceListController' => 'applications/drydock/controller/DrydockResourceListController.php',
863
865
'DrydockResourceListView' => 'applications/drydock/view/DrydockResourceListView.php',
864
866
'DrydockResourcePHIDType' => 'applications/drydock/phid/DrydockResourcePHIDType.php',
···
4562
4564
),
4563
4565
'DrydockLeaseController' => 'DrydockController',
4564
4566
'DrydockLeaseDatasource' => 'PhabricatorTypeaheadDatasource',
4567
+
'DrydockLeaseDestroyWorker' => 'DrydockWorker',
4565
4568
'DrydockLeaseListController' => 'DrydockLeaseController',
4566
4569
'DrydockLeaseListView' => 'AphrontView',
4567
4570
'DrydockLeasePHIDType' => 'PhabricatorPHIDType',
···
4595
4598
),
4596
4599
'DrydockResourceController' => 'DrydockController',
4597
4600
'DrydockResourceDatasource' => 'PhabricatorTypeaheadDatasource',
4601
+
'DrydockResourceDestroyWorker' => 'DrydockWorker',
4598
4602
'DrydockResourceListController' => 'DrydockResourceController',
4599
4603
'DrydockResourceListView' => 'AphrontView',
4600
4604
'DrydockResourcePHIDType' => 'PhabricatorPHIDType',
+26
src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php
+26
src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php
···
87
87
$exceptions);
88
88
}
89
89
90
+
public function destroyResource(
91
+
DrydockBlueprint $blueprint,
92
+
DrydockResource $resource) {
93
+
// We don't create anything when allocating hosts, so we don't need to do
94
+
// any cleanup here.
95
+
return;
96
+
}
97
+
90
98
public function canAcquireLeaseOnResource(
91
99
DrydockBlueprint $blueprint,
92
100
DrydockResource $resource,
···
108
116
->setActivateWhenAcquired(true)
109
117
->needSlotLock($this->getLeaseSlotLock($resource))
110
118
->acquireOnResource($resource);
119
+
}
120
+
121
+
public function didReleaseLease(
122
+
DrydockBlueprint $blueprint,
123
+
DrydockResource $resource,
124
+
DrydockLease $lease) {
125
+
// Almanac hosts stick around indefinitely so we don't need to recycle them
126
+
// if they don't have any leases.
127
+
return;
128
+
}
129
+
130
+
public function destroyLease(
131
+
DrydockBlueprint $blueprint,
132
+
DrydockResource $resource,
133
+
DrydockLease $lease) {
134
+
// We don't create anything when activating a lease, so we don't need to
135
+
// throw anything away.
136
+
return;
111
137
}
112
138
113
139
private function getLeaseSlotLock(DrydockResource $resource) {
+57
-27
src/applications/drydock/blueprint/DrydockBlueprintImplementation.php
+57
-27
src/applications/drydock/blueprint/DrydockBlueprintImplementation.php
···
67
67
DrydockResource $resource,
68
68
DrydockLease $lease);
69
69
70
+
71
+
/**
72
+
* @return void
73
+
* @task lease
74
+
*/
70
75
public function activateLease(
71
76
DrydockBlueprint $blueprint,
72
77
DrydockResource $resource,
···
74
79
throw new PhutilMethodNotImplementedException();
75
80
}
76
81
77
-
final public function releaseLease(
82
+
83
+
/**
84
+
* React to a lease being released.
85
+
*
86
+
* This callback is primarily useful for automatically releasing resources
87
+
* once all leases are released.
88
+
*
89
+
* @param DrydockBlueprint Blueprint which built the resource.
90
+
* @param DrydockResource Resource a lease was released on.
91
+
* @param DrydockLease Recently released lease.
92
+
* @return void
93
+
* @task lease
94
+
*/
95
+
abstract public function didReleaseLease(
78
96
DrydockBlueprint $blueprint,
79
97
DrydockResource $resource,
80
-
DrydockLease $lease) {
81
-
82
-
// TODO: This is all broken nonsense.
83
-
84
-
$scope = $this->pushActiveScope(null, $lease);
85
-
86
-
$released = false;
87
-
88
-
$lease->openTransaction();
89
-
$lease->beginReadLocking();
90
-
$lease->reload();
98
+
DrydockLease $lease);
91
99
92
-
if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACTIVE) {
93
-
$lease->release();
94
-
$lease->setStatus(DrydockLeaseStatus::STATUS_RELEASED);
95
-
$lease->save();
96
-
$released = true;
97
-
}
98
100
99
-
$lease->endReadLocking();
100
-
$lease->saveTransaction();
101
-
102
-
if (!$released) {
103
-
throw new Exception(pht('Unable to release lease: lease not active!'));
104
-
}
105
-
106
-
}
107
-
101
+
/**
102
+
* Destroy any temporary data associated with a lease.
103
+
*
104
+
* If a lease creates temporary state while held, destroy it here.
105
+
*
106
+
* @param DrydockBlueprint Blueprint which built the resource.
107
+
* @param DrydockResource Resource the lease is acquired on.
108
+
* @param DrydockLease The lease being destroyed.
109
+
* @return void
110
+
* @task lease
111
+
*/
112
+
abstract public function destroyLease(
113
+
DrydockBlueprint $blueprint,
114
+
DrydockResource $resource,
115
+
DrydockLease $lease);
108
116
109
117
110
118
/* -( Resource Allocation )------------------------------------------------ */
···
204
212
DrydockBlueprint $blueprint,
205
213
DrydockLease $lease);
206
214
215
+
216
+
/**
217
+
* @task resource
218
+
*/
207
219
public function activateResource(
208
220
DrydockBlueprint $blueprint,
209
221
DrydockResource $resource) {
210
222
throw new PhutilMethodNotImplementedException();
211
223
}
224
+
225
+
226
+
/**
227
+
* Destroy any temporary data associated with a resource.
228
+
*
229
+
* If a resource creates temporary state when allocated, destroy that state
230
+
* here. For example, you might shut down a virtual host or destroy a working
231
+
* copy on disk.
232
+
*
233
+
* @param DrydockBlueprint Blueprint which built the resource.
234
+
* @param DrydockResource Resource being destroyed.
235
+
* @return void
236
+
* @task resource
237
+
*/
238
+
abstract public function destroyResource(
239
+
DrydockBlueprint $blueprint,
240
+
DrydockResource $resource);
241
+
212
242
213
243
/* -( Resource Interfaces )------------------------------------------------ */
214
244
+40
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
+40
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
···
126
126
->activateResource();
127
127
}
128
128
129
+
public function destroyResource(
130
+
DrydockBlueprint $blueprint,
131
+
DrydockResource $resource) {
132
+
133
+
$lease = $this->loadHostLease($resource);
134
+
135
+
// Destroy the lease on the host.
136
+
$lease->releaseOnDestruction();
137
+
138
+
// Destroy the working copy on disk.
139
+
$command_type = DrydockCommandInterface::INTERFACE_TYPE;
140
+
$interface = $lease->getInterface($command_type);
141
+
142
+
$root_key = 'workingcopy.root';
143
+
$root = $resource->getAttribute($root_key);
144
+
if (strlen($root)) {
145
+
$interface->execx('rm -rf -- %s', $root);
146
+
}
147
+
}
148
+
129
149
public function activateLease(
130
150
DrydockBlueprint $blueprint,
131
151
DrydockResource $resource,
···
160
180
$argv);
161
181
162
182
$lease->activateOnResource($resource);
183
+
}
184
+
185
+
public function didReleaseLease(
186
+
DrydockBlueprint $blueprint,
187
+
DrydockResource $resource,
188
+
DrydockLease $lease) {
189
+
// We leave working copies around even if there are no leases on them,
190
+
// since the cost to maintain them is nearly zero but rebuilding them is
191
+
// moderately expensive and it's likely that they'll be reused.
192
+
return;
193
+
}
194
+
195
+
public function destroyLease(
196
+
DrydockBlueprint $blueprint,
197
+
DrydockResource $resource,
198
+
DrydockLease $lease) {
199
+
// When we activate a lease we just reset the working copy state and do
200
+
// not create any new state, so we don't need to do anything special when
201
+
// destroying a lease.
202
+
return;
163
203
}
164
204
165
205
public function getType() {
+11
-4
src/applications/drydock/management/DrydockManagementLeaseWorkflow.php
+11
-4
src/applications/drydock/management/DrydockManagementLeaseWorkflow.php
···
45
45
if ($attributes) {
46
46
$lease->setAttributes($attributes);
47
47
}
48
-
$lease
49
-
->queueForActivation()
50
-
->waitUntilActive();
48
+
$lease->queueForActivation();
49
+
50
+
echo tsprintf(
51
+
"%s\n",
52
+
pht('Waiting for daemons to activate lease...'));
53
+
54
+
$lease->waitUntilActive();
55
+
56
+
echo tsprintf(
57
+
"%s\n",
58
+
pht('Activated lease "%s".', $lease->getID()));
51
59
52
-
$console->writeOut("%s\n", pht('Acquired Lease %s', $lease->getID()));
53
60
return 0;
54
61
}
55
62
+31
-2
src/applications/drydock/storage/DrydockBlueprint.php
+31
-2
src/applications/drydock/storage/DrydockBlueprint.php
···
143
143
$resource);
144
144
}
145
145
146
+
147
+
/**
148
+
* @task resource
149
+
*/
150
+
public function destroyResource(DrydockResource $resource) {
151
+
$this->getImplementation()->destroyResource(
152
+
$this,
153
+
$resource);
154
+
return $this;
155
+
}
156
+
157
+
146
158
/* -( Acquiring Leases )--------------------------------------------------- */
147
159
148
160
···
188
200
/**
189
201
* @task lease
190
202
*/
191
-
public function releaseLease(
203
+
public function didReleaseLease(
204
+
DrydockResource $resource,
205
+
DrydockLease $lease) {
206
+
$this->getImplementation()->didReleaseLease(
207
+
$this,
208
+
$resource,
209
+
$lease);
210
+
return $this;
211
+
}
212
+
213
+
214
+
/**
215
+
* @task lease
216
+
*/
217
+
public function destroyLease(
192
218
DrydockResource $resource,
193
219
DrydockLease $lease) {
194
-
$this->getImplementation()->releaseLease($this, $resource, $lease);
220
+
$this->getImplementation()->destroyLease(
221
+
$this,
222
+
$resource,
223
+
$lease);
195
224
return $this;
196
225
}
197
226
+39
src/applications/drydock/worker/DrydockLeaseDestroyWorker.php
+39
src/applications/drydock/worker/DrydockLeaseDestroyWorker.php
···
1
+
<?php
2
+
3
+
final class DrydockLeaseDestroyWorker extends DrydockWorker {
4
+
5
+
protected function doWork() {
6
+
$lease_phid = $this->getTaskDataValue('leasePHID');
7
+
$lease = $this->loadLease($lease_phid);
8
+
$this->destroyLease($lease);
9
+
}
10
+
11
+
private function destroyLease(DrydockLease $lease) {
12
+
$status = $lease->getStatus();
13
+
14
+
switch ($status) {
15
+
case DrydockLeaseStatus::STATUS_RELEASED:
16
+
case DrydockLeaseStatus::STATUS_BROKEN:
17
+
break;
18
+
default:
19
+
throw new PhabricatorWorkerPermanentFailureException(
20
+
pht(
21
+
'Unable to destroy lease ("%s"), lease has the wrong '.
22
+
'status ("%s").',
23
+
$lease->getPHID(),
24
+
$status));
25
+
}
26
+
27
+
$resource = $lease->getResource();
28
+
$blueprint = $resource->getBlueprint();
29
+
30
+
$blueprint->destroyLease($resource, $lease);
31
+
32
+
// TODO: Rename DrydockLeaseStatus::STATUS_EXPIRED to STATUS_DESTROYED.
33
+
34
+
$lease
35
+
->setStatus(DrydockLeaseStatus::STATUS_EXPIRED)
36
+
->save();
37
+
}
38
+
39
+
}
+13
-2
src/applications/drydock/worker/DrydockLeaseUpdateWorker.php
+13
-2
src/applications/drydock/worker/DrydockLeaseUpdateWorker.php
···
53
53
DrydockSlotLock::releaseLocks($lease->getPHID());
54
54
$lease->saveTransaction();
55
55
56
-
// TODO: Hook for resource release behaviors.
57
-
// TODO: Schedule lease destruction.
56
+
PhabricatorWorker::scheduleTask(
57
+
'DrydockLeaseDestroyWorker',
58
+
array(
59
+
'leasePHID' => $lease->getPHID(),
60
+
),
61
+
array(
62
+
'objectPHID' => $lease->getPHID(),
63
+
));
64
+
65
+
$resource = $lease->getResource();
66
+
$blueprint = $resource->getBlueprint();
67
+
68
+
$blueprint->didReleaseLease($resource, $lease);
58
69
}
59
70
60
71
}
+35
src/applications/drydock/worker/DrydockResourceDestroyWorker.php
+35
src/applications/drydock/worker/DrydockResourceDestroyWorker.php
···
1
+
<?php
2
+
3
+
final class DrydockResourceDestroyWorker extends DrydockWorker {
4
+
5
+
protected function doWork() {
6
+
$resource_phid = $this->getTaskDataValue('resourcePHID');
7
+
$resource = $this->loadResource($resource_phid);
8
+
$this->destroyResource($resource);
9
+
}
10
+
11
+
private function destroyResource(DrydockResource $resource) {
12
+
$status = $resource->getStatus();
13
+
14
+
switch ($status) {
15
+
case DrydockResourceStatus::STATUS_CLOSED:
16
+
case DrydockResourceStatus::STATUS_BROKEN:
17
+
break;
18
+
default:
19
+
throw new PhabricatorWorkerPermanentFailureException(
20
+
pht(
21
+
'Unable to destroy resource ("%s"), resource has the wrong '.
22
+
'status ("%s").',
23
+
$resource->getPHID(),
24
+
$status));
25
+
}
26
+
27
+
$blueprint = $resource->getBlueprint();
28
+
$blueprint->destroyResource($resource);
29
+
30
+
$resource
31
+
->setStatus(DrydockResourceStatus::STATUS_DESTROYED)
32
+
->save();
33
+
}
34
+
35
+
}
+8
-1
src/applications/drydock/worker/DrydockResourceUpdateWorker.php
+8
-1
src/applications/drydock/worker/DrydockResourceUpdateWorker.php
···
86
86
$lease->scheduleUpdate();
87
87
}
88
88
89
-
// TODO: Schedule resource destruction.
89
+
PhabricatorWorker::scheduleTask(
90
+
'DrydockResourceDestroyWorker',
91
+
array(
92
+
'resourcePHID' => $resource->getPHID(),
93
+
),
94
+
array(
95
+
'objectPHID' => $resource->getPHID(),
96
+
));
90
97
}
91
98
92
99
}