@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Make various Drydock CLI/Allocator improvements
Summary:
- Remove EC2, RemoteHost, Application, etc., blueprints for now. They're very proof-of-concept and Blueprints are getting API changes I don't want to bother propagating for now. Leave the abstract base class and the LocalHost blueprint. I'll restore the more complicated ones once better foundations are in place.
- Remove the Allocate controller from the web UI. The original vision here was that you'd manually allocate resources in some cases, but it no longer makes sense to do so as all allocations come from leases now. This simplifies allocations and makes the rule for when we can clean up resources clear-cut (if a resource has no more active leases, it can be cleaned up). Instead, we'll build resources like the localhost and remote hosts lazily, when leases come in for them.
- Add some configuration to manage the localhost blueprint.
- Refactor `canAllocateResources()` into `isEnabled()` (for config checks) and `canAllocateMoreResources()` (for quota checks, e.g. too many resources are allocated already).
- Juggle some signatures to align better with a world where blueprints generally do allocate.
- Add some more logging and error handling.
- Fix an issue with log ordering.
Test Plan: Allocated some localhost leases.
Reviewers: btrahan
Reviewed By: btrahan
CC: aran
Maniphest Tasks: T2015
Differential Revision: https://secure.phabricator.com/D3902
+219
-516
+16
conf/default.conf.php
+16
conf/default.conf.php
···
1136
1136
1137
1137
// -- Drydock --------------------------------------------------------------- //
1138
1138
1139
+
// Drydock is used to allocate various software resources. For example, it
1140
+
// allocates working copies so continuous integration tests can be executed.
1141
+
// It needs at least one host to allocate these resources on.
1142
+
//
1143
+
// Set this option to true to let Drydock use the localhost for allocations.
1144
+
// This is the simplest configuration, but the least scalable. You MUST
1145
+
// disable this if you run daemons on more than one machine -- if you do not,
1146
+
// a daemon on machine A may allocate a resource locally, and then a daemon
1147
+
// on machine B may try to access it.
1148
+
'drydock.localhost.enabled' => true,
1149
+
1150
+
// If the localhost is available to Drydock, specify the path on disk where
1151
+
// Drydock should write files. You should create this directory and make sure
1152
+
// the user that the daemons run as has permission to write to it.
1153
+
'drydock.localhost.path' => '/var/drydock/',
1154
+
1139
1155
// If you want to use Drydock's builtin EC2 Blueprints, configure your AWS
1140
1156
// EC2 credentials here.
1141
1157
'amazon-ec2.access-key' => null,
-10
src/__phutil_library_map__.php
-10
src/__phutil_library_map__.php
···
412
412
'DiffusionView' => 'applications/diffusion/view/DiffusionView.php',
413
413
'DivinerListController' => 'applications/diviner/controller/DivinerListController.php',
414
414
'DrydockAllocatorWorker' => 'applications/drydock/worker/DrydockAllocatorWorker.php',
415
-
'DrydockApacheWebrootBlueprint' => 'applications/drydock/blueprint/webroot/DrydockApacheWebrootBlueprint.php',
416
415
'DrydockApacheWebrootInterface' => 'applications/drydock/interface/webroot/DrydockApacheWebrootInterface.php',
417
416
'DrydockBlueprint' => 'applications/drydock/blueprint/DrydockBlueprint.php',
418
417
'DrydockCommandInterface' => 'applications/drydock/interface/command/DrydockCommandInterface.php',
419
418
'DrydockConstants' => 'applications/drydock/constants/DrydockConstants.php',
420
419
'DrydockController' => 'applications/drydock/controller/DrydockController.php',
421
420
'DrydockDAO' => 'applications/drydock/storage/DrydockDAO.php',
422
-
'DrydockEC2HostBlueprint' => 'applications/drydock/blueprint/DrydockEC2HostBlueprint.php',
423
421
'DrydockInterface' => 'applications/drydock/interface/DrydockInterface.php',
424
422
'DrydockLease' => 'applications/drydock/storage/DrydockLease.php',
425
423
'DrydockLeaseListController' => 'applications/drydock/controller/DrydockLeaseListController.php',
···
433
431
'DrydockManagementLeaseWorkflow' => 'applications/drydock/management/DrydockManagementLeaseWorkflow.php',
434
432
'DrydockManagementWaitForLeaseWorkflow' => 'applications/drydock/management/DrydockManagementWaitForLeaseWorkflow.php',
435
433
'DrydockManagementWorkflow' => 'applications/drydock/management/DrydockManagementWorkflow.php',
436
-
'DrydockPhabricatorApplicationBlueprint' => 'applications/drydock/blueprint/application/DrydockPhabricatorApplicationBlueprint.php',
437
-
'DrydockRemoteHostBlueprint' => 'applications/drydock/blueprint/DrydockRemoteHostBlueprint.php',
438
434
'DrydockResource' => 'applications/drydock/storage/DrydockResource.php',
439
-
'DrydockResourceAllocateController' => 'applications/drydock/controller/DrydockResourceAllocateController.php',
440
435
'DrydockResourceListController' => 'applications/drydock/controller/DrydockResourceListController.php',
441
436
'DrydockResourceStatus' => 'applications/drydock/constants/DrydockResourceStatus.php',
442
437
'DrydockSSHCommandInterface' => 'applications/drydock/interface/command/DrydockSSHCommandInterface.php',
···
1649
1644
'DiffusionView' => 'AphrontView',
1650
1645
'DivinerListController' => 'PhabricatorController',
1651
1646
'DrydockAllocatorWorker' => 'PhabricatorWorker',
1652
-
'DrydockApacheWebrootBlueprint' => 'DrydockBlueprint',
1653
1647
'DrydockApacheWebrootInterface' => 'DrydockWebrootInterface',
1654
1648
'DrydockCommandInterface' => 'DrydockInterface',
1655
1649
'DrydockController' => 'PhabricatorController',
1656
1650
'DrydockDAO' => 'PhabricatorLiskDAO',
1657
-
'DrydockEC2HostBlueprint' => 'DrydockRemoteHostBlueprint',
1658
1651
'DrydockLease' => 'DrydockDAO',
1659
1652
'DrydockLeaseListController' => 'DrydockController',
1660
1653
'DrydockLeaseStatus' => 'DrydockConstants',
···
1667
1660
'DrydockManagementLeaseWorkflow' => 'DrydockManagementWorkflow',
1668
1661
'DrydockManagementWaitForLeaseWorkflow' => 'DrydockManagementWorkflow',
1669
1662
'DrydockManagementWorkflow' => 'PhutilArgumentWorkflow',
1670
-
'DrydockPhabricatorApplicationBlueprint' => 'DrydockBlueprint',
1671
-
'DrydockRemoteHostBlueprint' => 'DrydockBlueprint',
1672
1663
'DrydockResource' => 'DrydockDAO',
1673
-
'DrydockResourceAllocateController' => 'DrydockController',
1674
1664
'DrydockResourceListController' => 'DrydockController',
1675
1665
'DrydockResourceStatus' => 'DrydockConstants',
1676
1666
'DrydockSSHCommandInterface' => 'DrydockCommandInterface',
-1
src/applications/drydock/application/PhabricatorApplicationDrydock.php
-1
src/applications/drydock/application/PhabricatorApplicationDrydock.php
···
31
31
'/drydock/' => array(
32
32
'' => 'DrydockResourceListController',
33
33
'resource/' => 'DrydockResourceListController',
34
-
'resource/allocate/' => 'DrydockResourceAllocateController',
35
34
'lease/' => array(
36
35
'' => 'DrydockLeaseListController',
37
36
'(?P<id>[1-9]\d*)/' => 'DrydockLeaseViewController',
+50
-15
src/applications/drydock/blueprint/DrydockBlueprint.php
+50
-15
src/applications/drydock/blueprint/DrydockBlueprint.php
···
11
11
DrydockLease $lease,
12
12
$type);
13
13
14
-
protected function executeAcquireLease(
15
-
DrydockResource $resource,
16
-
DrydockLease $lease) {
17
-
return;
14
+
abstract public function isEnabled();
15
+
16
+
public function getBlueprintClass() {
17
+
return get_class($this);
18
18
}
19
+
20
+
public function canAllocateMoreResources(array $pool) {
21
+
return true;
22
+
}
23
+
24
+
abstract protected function executeAllocateResource(DrydockLease $lease);
25
+
26
+
abstract protected function executeAcquireLease(
27
+
DrydockResource $resource,
28
+
DrydockLease $lease);
19
29
20
30
final public function acquireLease(
21
31
DrydockResource $resource,
···
74
84
$log->save();
75
85
}
76
86
77
-
public function canAllocateResources() {
78
-
return false;
79
-
}
80
-
81
-
protected function executeAllocateResource(DrydockLease $lease) {
82
-
throw new Exception("This blueprint can not allocate resources!");
83
-
}
84
-
85
87
final public function allocateResource(DrydockLease $lease) {
86
88
$this->activeLease = $lease;
87
89
$this->activeResource = null;
88
90
89
-
$this->log('Allocating Resource');
91
+
$this->log(
92
+
pht(
93
+
"Blueprint '%s': Allocating Resource for '%s'",
94
+
$this->getBlueprintClass(),
95
+
$lease->getLeaseName()));
90
96
91
97
try {
92
98
$resource = $this->executeAllocateResource($lease);
99
+
$this->validateAllocatedResource($resource);
93
100
} catch (Exception $ex) {
94
101
$this->logException($ex);
95
102
$this->activeResource = null;
···
128
135
129
136
protected function newResourceTemplate($name) {
130
137
$resource = new DrydockResource();
131
-
$resource->setBlueprintClass(get_class($this));
138
+
$resource->setBlueprintClass($this->getBlueprintClass());
132
139
$resource->setType($this->getType());
133
140
$resource->setStatus(DrydockResourceStatus::STATUS_PENDING);
134
141
$resource->setName($name);
135
142
$resource->save();
136
143
137
144
$this->activeResource = $resource;
138
-
$this->log('New Template');
145
+
$this->log(
146
+
pht(
147
+
"Blueprint '%s': Created New Template",
148
+
$this->getBlueprintClass()));
139
149
140
150
return $resource;
141
151
}
142
152
153
+
/**
154
+
* Sanity checks that the blueprint is implemented properly.
155
+
*/
156
+
private function validateAllocatedResource($resource) {
157
+
$blueprint = $this->getBlueprintClass();
158
+
159
+
if (!($resource instanceof DrydockResource)) {
160
+
throw new Exception(
161
+
"Blueprint '{$blueprint}' is not properly implemented: ".
162
+
"executeAllocateResource() must return an object of type ".
163
+
"DrydockResource or throw, but returned something else.");
164
+
}
165
+
166
+
$current_status = $resource->getStatus();
167
+
$req_status = DrydockResourceStatus::STATUS_OPEN;
168
+
if ($current_status != $req_status) {
169
+
$current_name = DrydockResourceStatus::getNameForStatus($current_status);
170
+
$req_name = DrydockResourceStatus::getNameForStatus($req_status);
171
+
throw new Exception(
172
+
"Blueprint '{$blueprint}' is not properly implemented: ".
173
+
"executeAllocateResource() must return a DrydockResource with ".
174
+
"status '{$req_name}', but returned one with status ".
175
+
"'{$current_name}'.");
176
+
}
177
+
}
143
178
144
179
}
-147
src/applications/drydock/blueprint/DrydockEC2HostBlueprint.php
-147
src/applications/drydock/blueprint/DrydockEC2HostBlueprint.php
···
1
-
<?php
2
-
3
-
final class DrydockEC2HostBlueprint extends DrydockRemoteHostBlueprint {
4
-
5
-
public function canAllocateResources() {
6
-
return true;
7
-
}
8
-
9
-
public function executeAllocateResource(DrydockLease $lease) {
10
-
$resource = $this->newResourceTemplate('EC2 Host');
11
-
12
-
$resource->setStatus(DrydockResourceStatus::STATUS_ALLOCATING);
13
-
$resource->save();
14
-
15
-
$xml = $this->executeEC2Query(
16
-
'RunInstances',
17
-
array(
18
-
'ImageId' => 'ami-c7c99982',
19
-
'MinCount' => 1,
20
-
'MaxCount' => 1,
21
-
'KeyName' => 'ec2wc',
22
-
'SecurityGroupId.1' => 'sg-6bffff2e',
23
-
'InstanceType' => 't1.micro',
24
-
));
25
-
26
-
$instance_id = (string)$xml->instancesSet[0]->item[0]->instanceId[0];
27
-
28
-
$this->log("Started Instance: {$instance_id}");
29
-
$resource->setAttribute('instance.id', $instance_id);
30
-
$resource->save();
31
-
32
-
$n = 1;
33
-
do {
34
-
$xml = $this->executeEC2Query(
35
-
'DescribeInstances',
36
-
array(
37
-
'InstanceId.1' => $instance_id,
38
-
));
39
-
40
-
$instance = $xml->reservationSet[0]->item[0]->instancesSet[0]->item[0];
41
-
42
-
$state = (string)$instance->instanceState[0]->name;
43
-
44
-
if ($state == 'pending') {
45
-
sleep(min($n++, 15));
46
-
} else if ($state == 'running') {
47
-
break;
48
-
} else {
49
-
$this->log("EC2 host reported in unknown state '{$state}'.");
50
-
51
-
$resource->setStatus(DrydockResourceStatus::STATUS_BROKEN);
52
-
$resource->save();
53
-
}
54
-
} while (true);
55
-
56
-
$this->log('Waiting for Init');
57
-
58
-
$n = 1;
59
-
do {
60
-
$xml = $this->executeEC2Query(
61
-
'DescribeInstanceStatus',
62
-
array(
63
-
'InstanceId' => $instance_id,
64
-
));
65
-
66
-
$item = $xml->instanceStatusSet[0]->item[0];
67
-
68
-
$system_status = (string)$item->systemStatus->status[0];
69
-
$instance_status = (string)$item->instanceStatus->status[0];
70
-
71
-
if (($system_status == 'initializing') ||
72
-
($instance_status == 'initializing')) {
73
-
sleep(min($n++, 15));
74
-
} else if (($system_status == 'ok') &&
75
-
($instance_status == 'ok')) {
76
-
break;
77
-
} else {
78
-
$this->log(
79
-
"EC2 system and instance status in bad states: ".
80
-
"'{$system_status}', '{$instance_status}'.");
81
-
82
-
$resource->setStatus(DrydockResourceStatus::STATUS_BROKEN);
83
-
$resource->save();
84
-
}
85
-
} while (true);
86
-
87
-
$resource->setAttributes(
88
-
array(
89
-
'host' => (string)$instance->dnsName,
90
-
'user' => 'ec2-user',
91
-
'ssh-keyfile' => '/Users/epriestley/.ssh/id_ec2w',
92
-
));
93
-
$resource->setName($resource->getName().' ('.$instance->dnsName.')');
94
-
$resource->save();
95
-
96
-
$this->log('Waiting for SSH');
97
-
98
-
// SSH isn't immediately responsive, so wait for it to actually come up.
99
-
$cmd = $this->getInterface($resource, new DrydockLease(), 'command');
100
-
$n = 1;
101
-
do {
102
-
list($err) = $cmd->exec('true');
103
-
if ($err) {
104
-
sleep(min($n++, 15));
105
-
} else {
106
-
break;
107
-
}
108
-
} while (true);
109
-
110
-
$this->log('SSH OK');
111
-
112
-
$resource->setStatus(DrydockResourceStatus::STATUS_OPEN);
113
-
$resource->save();
114
-
115
-
return $resource;
116
-
}
117
-
118
-
public function getInterface(
119
-
DrydockResource $resource,
120
-
DrydockLease $lease,
121
-
$type) {
122
-
123
-
switch ($type) {
124
-
case 'command':
125
-
$ssh = new DrydockSSHCommandInterface();
126
-
$ssh->setConfiguration(
127
-
array(
128
-
'host' => $resource->getAttribute('host'),
129
-
'user' => $resource->getAttribute('user'),
130
-
'ssh-keyfile' => $resource->getAttribute('ssh-keyfile'),
131
-
));
132
-
return $ssh;
133
-
}
134
-
135
-
throw new Exception("No interface of type '{$type}'.");
136
-
}
137
-
138
-
private function executeEC2Query($action, array $params) {
139
-
$future = new PhutilAWSEC2Future();
140
-
$future->setAWSKeys(
141
-
PhabricatorEnv::getEnvConfig('amazon-ec2.access-key'),
142
-
PhabricatorEnv::getEnvConfig('amazon-ec2.secret-key'));
143
-
$future->setRawAWSQuery($action, $params);
144
-
return $future->resolve();
145
-
}
146
-
147
-
}
+39
src/applications/drydock/blueprint/DrydockLocalHostBlueprint.php
+39
src/applications/drydock/blueprint/DrydockLocalHostBlueprint.php
···
2
2
3
3
final class DrydockLocalHostBlueprint extends DrydockBlueprint {
4
4
5
+
public function isEnabled() {
6
+
return PhabricatorEnv::getEnvConfig('drydock.localhost.enabled');
7
+
}
8
+
9
+
public function canAllocateMoreResources(array $pool) {
10
+
assert_instances_of($pool, 'DrydockResource');
11
+
12
+
// The localhost can be allocated only once.
13
+
foreach ($pool as $resource) {
14
+
if ($resource->getBlueprintClass() == $this->getBlueprintClass()) {
15
+
return false;
16
+
}
17
+
}
18
+
19
+
return true;
20
+
}
21
+
22
+
protected function executeAllocateResource(DrydockLease $lease) {
23
+
$path = PhabricatorEnv::getEnvConfig('drydock.localhost.path');
24
+
if (!Filesystem::pathExists($path)) {
25
+
throw new Exception(
26
+
"Path '{$path}' does not exist!");
27
+
}
28
+
Filesystem::assertIsDirectory($path);
29
+
Filesystem::assertWritable($path);
30
+
31
+
$resource = $this->newResourceTemplate('localhost');
32
+
$resource->setStatus(DrydockResourceStatus::STATUS_OPEN);
33
+
$resource->save();
34
+
35
+
return $resource;
36
+
}
37
+
38
+
protected function executeAcquireLease(
39
+
DrydockResource $resource,
40
+
DrydockLease $lease) {
41
+
return;
42
+
}
43
+
5
44
public function getType() {
6
45
return 'host';
7
46
}
-32
src/applications/drydock/blueprint/DrydockRemoteHostBlueprint.php
-32
src/applications/drydock/blueprint/DrydockRemoteHostBlueprint.php
···
1
-
<?php
2
-
3
-
/**
4
-
* TODO: Is this concrete-extensible?
5
-
*/
6
-
class DrydockRemoteHostBlueprint extends DrydockBlueprint {
7
-
8
-
public function getType() {
9
-
return 'host';
10
-
}
11
-
12
-
public function getInterface(
13
-
DrydockResource $resource,
14
-
DrydockLease $lease,
15
-
$type) {
16
-
17
-
switch ($type) {
18
-
case 'command':
19
-
$ssh = new DrydockSSHCommandInterface();
20
-
$ssh->setConfiguration(
21
-
array(
22
-
'host' => 'secure.phabricator.com',
23
-
'user' => 'ec2-user',
24
-
'ssh-keyfile' => '/Users/epriestley/.ssh/id_ec2w',
25
-
));
26
-
return $ssh;
27
-
}
28
-
29
-
throw new Exception("No interface of type '{$type}'.");
30
-
}
31
-
32
-
}
-52
src/applications/drydock/blueprint/application/DrydockPhabricatorApplicationBlueprint.php
-52
src/applications/drydock/blueprint/application/DrydockPhabricatorApplicationBlueprint.php
···
1
-
<?php
2
-
3
-
abstract class DrydockPhabricatorApplicationBlueprint
4
-
extends DrydockBlueprint {
5
-
6
-
public function getType() {
7
-
return 'application';
8
-
}
9
-
10
-
public function canAllocateResources() {
11
-
return true;
12
-
}
13
-
14
-
public function executeAllocateResource(DrydockLease $lease) {
15
-
16
-
$resource = $this->newResourceTemplate('Phabricator');
17
-
18
-
$resource->setStatus(DrydockResourceStatus::STATUS_ALLOCATING);
19
-
$resource->save();
20
-
21
-
$host = id(new DrydockLease())
22
-
->setResourceType('host')
23
-
->queueForActivation();
24
-
25
-
$cmd = $host->waitUntilActive()->getInterface('command');
26
-
27
-
$cmd->execx(<<<EOINSTALL
28
-
yum install git &&
29
-
mkdir -p /opt/drydock &&
30
-
cd /opt/drydock &&
31
-
git clone git://github.com/facebook/libphutil.git &&
32
-
git clone git://github.com/facebook/arcanist.git &&
33
-
git clone git://github.com/facebook/phabricator.git
34
-
EOINSTALL
35
-
);
36
-
37
-
$resource->setStatus(DrydockResourceStatus::STATUS_OPEN);
38
-
$resource->save();
39
-
40
-
return $resource;
41
-
}
42
-
43
-
public function getInterface(
44
-
DrydockResource $resource,
45
-
DrydockLease $lease,
46
-
$type) {
47
-
48
-
49
-
throw new Exception("No interface of type '{$type}'.");
50
-
}
51
-
52
-
}
-113
src/applications/drydock/blueprint/webroot/DrydockApacheWebrootBlueprint.php
-113
src/applications/drydock/blueprint/webroot/DrydockApacheWebrootBlueprint.php
···
1
-
<?php
2
-
3
-
final class DrydockApacheWebrootBlueprint
4
-
extends DrydockBlueprint {
5
-
6
-
public function getType() {
7
-
return 'webroot';
8
-
}
9
-
10
-
public function canAllocateResources() {
11
-
return true;
12
-
}
13
-
14
-
public function executeAcquireLease(
15
-
DrydockResource $resource,
16
-
DrydockLease $lease) {
17
-
18
-
$key = Filesystem::readRandomCharacters(12);
19
-
20
-
$ports = $resource->getAttribute('ports', array());
21
-
for ($ii = 2000; ; $ii++) {
22
-
if (empty($ports[$ii])) {
23
-
$ports[$ii] = $lease->getID();
24
-
$port = $ii;
25
-
break;
26
-
}
27
-
}
28
-
$resource->setAttribute('ports', $ports);
29
-
$resource->save();
30
-
31
-
$host = $resource->getAttribute('host');
32
-
33
-
$lease->setAttribute('port', $port);
34
-
$lease->setAttribute('key', $key);
35
-
$lease->save();
36
-
37
-
$config = <<<EOCONFIG
38
-
39
-
Listen *:{$port}
40
-
<VirtualHost *:{$port}>
41
-
DocumentRoot /opt/drydock/webroot/{$key}/
42
-
ServerName {$host}
43
-
</VirtualHost>
44
-
EOCONFIG;
45
-
46
-
$cmd = $this->getInterface($resource, $lease, 'command');
47
-
$cmd->execx(<<<EOSETUP
48
-
sudo mkdir -p %s &&
49
-
sudo sh -c %s &&
50
-
sudo /etc/init.d/httpd restart
51
-
EOSETUP
52
-
,
53
-
"/opt/drydock/webroot/{$key}/",
54
-
csprintf(
55
-
'echo %s > %s',
56
-
$config,
57
-
"/etc/httpd/conf.d/drydock-{$key}.conf"));
58
-
59
-
$lease->setAttribute('uri', "http://{$host}:{$port}/");
60
-
$lease->save();
61
-
}
62
-
63
-
public function executeAllocateResource(DrydockLease $lease) {
64
-
65
-
$resource = $this->newResourceTemplate('Apache');
66
-
67
-
$resource->setStatus(DrydockResourceStatus::STATUS_ALLOCATING);
68
-
$resource->save();
69
-
70
-
$allocator = $this->getAllocator('host');
71
-
$host = $allocator->allocate();
72
-
73
-
$cmd = $host->waitUntilActive()->getInterface('command');
74
-
75
-
$cmd->execx(<<<EOINSTALL
76
-
(yes | sudo yum install httpd) && sudo mkdir -p /opt/drydock/webroot/
77
-
EOINSTALL
78
-
);
79
-
80
-
$resource->setAttribute('lease.host', $host->getID());
81
-
$resource->setAttribute('host', $host->getResource()->getAttribute('host'));
82
-
83
-
$resource->setStatus(DrydockResourceStatus::STATUS_OPEN);
84
-
$resource->save();
85
-
86
-
return $resource;
87
-
}
88
-
89
-
public function getInterface(
90
-
DrydockResource $resource,
91
-
DrydockLease $lease,
92
-
$type) {
93
-
94
-
switch ($type) {
95
-
case 'webroot':
96
-
$iface = new DrydockApacheWebrootInterface();
97
-
$iface->setConfiguration(
98
-
array(
99
-
'uri' => $lease->getAttribute('uri'),
100
-
));
101
-
return $iface;
102
-
case 'command':
103
-
$host_lease_id = $resource->getAttribute('lease.host');
104
-
$host_lease = id(new DrydockLease())->load($host_lease_id);
105
-
$host_lease->attachResource($host_lease->loadResource());
106
-
return $host_lease->getInterface($type);
107
-
}
108
-
109
-
110
-
throw new Exception("No interface of type '{$type}'.");
111
-
}
112
-
113
-
}
+4
-6
src/applications/drydock/constants/DrydockResourceStatus.php
+4
-6
src/applications/drydock/constants/DrydockResourceStatus.php
···
3
3
final class DrydockResourceStatus extends DrydockConstants {
4
4
5
5
const STATUS_PENDING = 0;
6
-
const STATUS_ALLOCATING = 1;
7
-
const STATUS_OPEN = 2;
8
-
const STATUS_CLOSED = 3;
9
-
const STATUS_BROKEN = 4;
10
-
const STATUS_DESTROYED = 5;
6
+
const STATUS_OPEN = 1;
7
+
const STATUS_CLOSED = 2;
8
+
const STATUS_BROKEN = 3;
9
+
const STATUS_DESTROYED = 4;
11
10
12
11
public static function getNameForStatus($status) {
13
12
static $map = array(
14
13
self::STATUS_PENDING => 'Pending',
15
-
self::STATUS_ALLOCATING => 'Pending',
16
14
self::STATUS_OPEN => 'Open',
17
15
self::STATUS_CLOSED => 'Closed',
18
16
self::STATUS_BROKEN => 'Broken',
-119
src/applications/drydock/controller/DrydockResourceAllocateController.php
-119
src/applications/drydock/controller/DrydockResourceAllocateController.php
···
1
-
<?php
2
-
3
-
final class DrydockResourceAllocateController extends DrydockController {
4
-
5
-
public function processRequest() {
6
-
$request = $this->getRequest();
7
-
$user = $request->getUser();
8
-
9
-
$resource = new DrydockResource();
10
-
11
-
$json = new PhutilJSON();
12
-
13
-
$err_attributes = true;
14
-
$err_capabilities = true;
15
-
16
-
$json_attributes = $json->encodeFormatted($resource->getAttributes());
17
-
$json_capabilities = $json->encodeFormatted($resource->getCapabilities());
18
-
19
-
$errors = array();
20
-
21
-
if ($request->isFormPost()) {
22
-
$raw_attributes = $request->getStr('attributes');
23
-
$attributes = json_decode($raw_attributes, true);
24
-
if (!is_array($attributes)) {
25
-
$err_attributes = 'Invalid';
26
-
$errors[] = 'Enter attributes as a valid JSON object.';
27
-
$json_attributes = $raw_attributes;
28
-
} else {
29
-
$resource->setAttributes($attributes);
30
-
$json_attributes = $json->encodeFormatted($attributes);
31
-
$err_attributes = null;
32
-
}
33
-
34
-
$raw_capabilities = $request->getStr('capabilities');
35
-
$capabilities = json_decode($raw_capabilities, true);
36
-
if (!is_array($capabilities)) {
37
-
$err_capabilities = 'Invalid';
38
-
$errors[] = 'Enter capabilities as a valid JSON object.';
39
-
$json_capabilities = $raw_capabilities;
40
-
} else {
41
-
$resource->setCapabilities($capabilities);
42
-
$json_capabilities = $json->encodeFormatted($capabilities);
43
-
$err_capabilities = null;
44
-
}
45
-
46
-
$resource->setBlueprintClass($request->getStr('blueprint'));
47
-
$resource->setType($resource->getBlueprint()->getType());
48
-
$resource->setOwnerPHID($user->getPHID());
49
-
$resource->setName($request->getStr('name'));
50
-
51
-
if (!$errors) {
52
-
$resource->save();
53
-
return id(new AphrontRedirectResponse())
54
-
->setURI('/drydock/resource/');
55
-
}
56
-
}
57
-
58
-
$error_view = null;
59
-
if ($errors) {
60
-
$error_view = new AphrontErrorView();
61
-
$error_view->setTitle('Form Errors');
62
-
$error_view->setErrors($errors);
63
-
}
64
-
65
-
66
-
$blueprints = id(new PhutilSymbolLoader())
67
-
->setType('class')
68
-
->setAncestorClass('DrydockBlueprint')
69
-
->selectAndLoadSymbols();
70
-
$blueprints = ipull($blueprints, 'name', 'name');
71
-
$panel = new AphrontPanelView();
72
-
$panel->setWidth(AphrontPanelView::WIDTH_FORM);
73
-
$panel->setHeader('Allocate Drydock Resource');
74
-
75
-
$form = id(new AphrontFormView())
76
-
->setUser($request->getUser())
77
-
->appendChild(
78
-
id(new AphrontFormTextControl())
79
-
->setLabel('Name')
80
-
->setName('name')
81
-
->setValue($resource->getName()))
82
-
->appendChild(
83
-
id(new AphrontFormSelectControl())
84
-
->setLabel('Blueprint')
85
-
->setOptions($blueprints)
86
-
->setName('blueprint')
87
-
->setValue($resource->getBlueprintClass()))
88
-
->appendChild(
89
-
id(new AphrontFormTextAreaControl())
90
-
->setLabel('Attributes')
91
-
->setName('attributes')
92
-
->setValue($json_attributes)
93
-
->setError($err_attributes)
94
-
->setCaption('Specify attributes in JSON.'))
95
-
->appendChild(
96
-
id(new AphrontFormTextAreaControl())
97
-
->setLabel('Capabilities')
98
-
->setName('capabilities')
99
-
->setValue($json_capabilities)
100
-
->setError($err_capabilities)
101
-
->setCaption('Specify capabilities in JSON.'))
102
-
->appendChild(
103
-
id(new AphrontFormSubmitControl())
104
-
->setValue('Allocate Resource'));
105
-
106
-
$panel->appendChild($form);
107
-
108
-
return $this->buildStandardPageResponse(
109
-
array(
110
-
$error_view,
111
-
$panel,
112
-
),
113
-
array(
114
-
'title' => 'Allocate Resource',
115
-
));
116
-
117
-
}
118
-
119
-
}
-9
src/applications/drydock/controller/DrydockResourceListController.php
-9
src/applications/drydock/controller/DrydockResourceListController.php
···
57
57
$panel = new AphrontPanelView();
58
58
$panel->setHeader('Drydock Resources');
59
59
60
-
$panel->addButton(
61
-
phutil_render_tag(
62
-
'a',
63
-
array(
64
-
'href' => '/drydock/resource/allocate/',
65
-
'class' => 'green button',
66
-
),
67
-
'Allocate Resource'));
68
-
69
60
$panel->appendChild($table);
70
61
$panel->appendChild($pager);
71
62
+1
-1
src/applications/drydock/query/DrydockLogQuery.php
+1
-1
src/applications/drydock/query/DrydockLogQuery.php
···
75
75
private function buildOrderClause(AphrontDatabaseConnection $conn_r) {
76
76
switch ($this->order) {
77
77
case self::ORDER_EPOCH:
78
-
return 'ORDER BY log.epoch DESC';
78
+
return 'ORDER BY log.epoch DESC, log.id DESC';
79
79
case self::ORDER_ID:
80
80
return 'ORDER BY id ASC';
81
81
default:
+8
-2
src/applications/drydock/storage/DrydockLease.php
+8
-2
src/applications/drydock/storage/DrydockLease.php
···
2
2
3
3
final class DrydockLease extends DrydockDAO {
4
4
5
-
protected $phid;
6
5
protected $resourceID;
7
6
protected $resourceType;
8
7
protected $until;
···
12
11
protected $taskID;
13
12
14
13
private $resource;
14
+
15
+
public function getLeaseName() {
16
+
return pht('Lease %d', $this->getID());
17
+
}
15
18
16
19
public function getConfiguration() {
17
20
return array(
···
112
115
assert_instances_of($leases, 'DrydockLease');
113
116
114
117
$task_ids = array_filter(mpull($leases, 'getTaskID'));
118
+
115
119
PhabricatorWorker::waitForTasks($task_ids);
116
120
117
121
$unresolved = $leases;
···
123
127
unset($unresolved[$key]);
124
128
break;
125
129
case DrydockLeaseStatus::STATUS_RELEASED:
130
+
throw new Exception("Lease has already been released!");
126
131
case DrydockLeaseStatus::STATUS_EXPIRED:
132
+
throw new Exception("Lease has already expired!");
127
133
case DrydockLeaseStatus::STATUS_BROKEN:
128
-
throw new Exception("Lease will never become active!");
134
+
throw new Exception("Lease has been broken!");
129
135
case DrydockLeaseStatus::STATUS_PENDING:
130
136
break;
131
137
}
+86
-9
src/applications/drydock/worker/DrydockAllocatorWorker.php
+86
-9
src/applications/drydock/worker/DrydockAllocatorWorker.php
···
2
2
3
3
final class DrydockAllocatorWorker extends PhabricatorWorker {
4
4
5
+
private $lease;
6
+
7
+
public function getMaximumRetryCount() {
8
+
// TODO: Allow Drydock allocations to retry. For now, every failure is
9
+
// permanent and most of them are because I am bad at programming, so fail
10
+
// fast rather than ending up in limbo.
11
+
return 0;
12
+
}
13
+
14
+
private function loadLease() {
15
+
if (empty($this->lease)) {
16
+
$lease = id(new DrydockLease())->load($this->getTaskData());
17
+
if (!$lease) {
18
+
throw new PhabricatorWorkerPermanentFailureException(
19
+
"No such lease!");
20
+
}
21
+
$this->lease = $lease;
22
+
}
23
+
return $this->lease;
24
+
}
25
+
26
+
private function log($message) {
27
+
DrydockBlueprint::writeLog(
28
+
null,
29
+
$this->loadLease(),
30
+
$message);
31
+
}
32
+
5
33
protected function doWork() {
6
-
$lease_id = $this->getTaskData();
34
+
$lease = $this->loadLease();
35
+
$this->log('Allocating Lease');
7
36
8
-
$lease = id(new DrydockLease())->load($lease_id);
9
-
if (!$lease) {
10
-
return;
37
+
try {
38
+
$this->allocateLease($lease);
39
+
} catch (Exception $ex) {
40
+
41
+
// TODO: We should really do this when archiving the task, if we've
42
+
// suffered a permanent failure. But we don't have hooks for that yet
43
+
// and always fail after the first retry right now, so this is
44
+
// functionally equivalent.
45
+
$lease->reload();
46
+
if ($lease->getStatus() == DrydockLeaseStatus::STATUS_PENDING) {
47
+
$lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN);
48
+
$lease->save();
49
+
}
50
+
51
+
throw $ex;
11
52
}
53
+
}
12
54
55
+
private function allocateLease(DrydockLease $lease) {
13
56
$type = $lease->getResourceType();
14
57
15
-
$candidates = id(new DrydockResource())->loadAllWhere(
58
+
$pool = id(new DrydockResource())->loadAllWhere(
16
59
'type = %s AND status = %s',
17
60
$lease->getResourceType(),
18
61
DrydockResourceStatus::STATUS_OPEN);
62
+
63
+
$this->log(
64
+
pht('Found %d Open Resource(s)', count($pool)));
65
+
66
+
$candidates = array();
67
+
foreach ($pool as $key => $candidate) {
68
+
try {
69
+
$candidate->getBlueprint();
70
+
} catch (Exception $ex) {
71
+
unset($pool[$key]);
72
+
}
73
+
74
+
// TODO: Filter candidates according to ability to satisfy the lease.
75
+
76
+
$candidates[] = $candidate;
77
+
}
78
+
79
+
$this->log(
80
+
pht('%d Open Resource(s) Remain', count($candidates)));
19
81
20
82
if ($candidates) {
21
83
shuffle($candidates);
22
84
$resource = head($candidates);
23
85
} else {
24
86
$blueprints = DrydockBlueprint::getAllBlueprintsForResource($type);
87
+
88
+
$this->log(
89
+
pht('Found %d Blueprints', count($blueprints)));
90
+
25
91
foreach ($blueprints as $key => $blueprint) {
26
-
if (!$blueprint->canAllocateResources()) {
92
+
if (!$blueprint->isEnabled()) {
93
+
unset($blueprints[$key]);
94
+
continue;
95
+
}
96
+
}
97
+
98
+
$this->log(
99
+
pht('%d Blueprints Enabled', count($blueprints)));
100
+
101
+
foreach ($blueprints as $key => $blueprint) {
102
+
if (!$blueprint->canAllocateMoreResources($pool)) {
27
103
unset($blueprints[$key]);
28
104
continue;
29
105
}
30
106
}
31
107
108
+
$this->log(
109
+
pht('%d Blueprints Can Allocate', count($blueprints)));
110
+
32
111
if (!$blueprints) {
33
112
$lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN);
34
113
$lease->save();
35
114
36
-
DrydockBlueprint::writeLog(
37
-
null,
38
-
$lease,
115
+
$this->log(
39
116
"There are no resources of type '{$type}' available, and no ".
40
117
"blueprints which can allocate new ones.");
41
118
+15
src/infrastructure/daemon/workers/PhabricatorWorker.php
+15
src/infrastructure/daemon/workers/PhabricatorWorker.php
···
131
131
}
132
132
133
133
$task = head($tasks)->executeTask();
134
+
135
+
$ex = $task->getExecutionException();
136
+
if ($ex) {
137
+
throw $ex;
138
+
}
139
+
}
140
+
141
+
$tasks = id(new PhabricatorWorkerArchiveTask())->loadAllWhere(
142
+
'id IN (%Ld)',
143
+
$task_ids);
144
+
145
+
foreach ($tasks as $task) {
146
+
if ($task->getResult() != PhabricatorWorkerArchiveTask::RESULT_SUCCESS) {
147
+
throw new Exception("Task ".$task->getID()." failed!");
148
+
}
134
149
}
135
150
}
136
151