@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Make various Drydock improvements
Summary:
Tightens up a bunch of stuff:
- In `drydock lease`, pull and print logs so the user can see what's happening.
- Remove `DrydockAllocator`, which was a dumb class that did nothing. Move the tiny amount of logic it held directly to `DrydockLease`.
- Move `resourceType` from worker task metadata directly to `DrydockLease`. Other things (like the web UI) can be more informative with this information available.
- Pass leases to `allocateResource()`. We always allocate in response to a lease activation request, and the lease often has vital information. This also allows us to associate logs with leases correctly.
Test Plan: Ran `drydock lease --type host` and saw it perform a host allocation in EC2.
Reviewers: btrahan
Reviewed By: btrahan
CC: aran
Maniphest Tasks: T2015
Differential Revision: https://secure.phabricator.com/D3870
+115
-109
+2
resources/sql/patches/drydockresourcetype.sql
+2
resources/sql/patches/drydockresourcetype.sql
+1
-2
src/__phutil_library_map__.php
+1
-2
src/__phutil_library_map__.php
···
411
411
'DiffusionURITestCase' => 'applications/diffusion/request/__tests__/DiffusionURITestCase.php',
412
412
'DiffusionView' => 'applications/diffusion/view/DiffusionView.php',
413
413
'DivinerListController' => 'applications/diviner/controller/DivinerListController.php',
414
-
'DrydockAllocator' => 'applications/drydock/allocator/DrydockAllocator.php',
415
-
'DrydockAllocatorWorker' => 'applications/drydock/allocator/DrydockAllocatorWorker.php',
414
+
'DrydockAllocatorWorker' => 'applications/drydock/worker/DrydockAllocatorWorker.php',
416
415
'DrydockApacheWebrootBlueprint' => 'applications/drydock/blueprint/webroot/DrydockApacheWebrootBlueprint.php',
417
416
'DrydockApacheWebrootInterface' => 'applications/drydock/interface/webroot/DrydockApacheWebrootInterface.php',
418
417
'DrydockBlueprint' => 'applications/drydock/blueprint/DrydockBlueprint.php',
-59
src/applications/drydock/allocator/DrydockAllocator.php
-59
src/applications/drydock/allocator/DrydockAllocator.php
···
1
-
<?php
2
-
3
-
/*
4
-
* Copyright 2012 Facebook, Inc.
5
-
*
6
-
* Licensed under the Apache License, Version 2.0 (the "License");
7
-
* you may not use this file except in compliance with the License.
8
-
* You may obtain a copy of the License at
9
-
*
10
-
* http://www.apache.org/licenses/LICENSE-2.0
11
-
*
12
-
* Unless required by applicable law or agreed to in writing, software
13
-
* distributed under the License is distributed on an "AS IS" BASIS,
14
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
-
* See the License for the specific language governing permissions and
16
-
* limitations under the License.
17
-
*/
18
-
19
-
final class DrydockAllocator {
20
-
21
-
private $resourceType;
22
-
private $lease;
23
-
24
-
public function setResourceType($resource_type) {
25
-
$this->resourceType = $resource_type;
26
-
return $this;
27
-
}
28
-
29
-
public function getResourceType() {
30
-
return $this->resourceType;
31
-
}
32
-
33
-
public function getPendingLease() {
34
-
if (!$this->lease) {
35
-
$lease = new DrydockLease();
36
-
$lease->setStatus(DrydockLeaseStatus::STATUS_PENDING);
37
-
$lease->save();
38
-
39
-
$this->lease = $lease;
40
-
}
41
-
return $lease;
42
-
}
43
-
44
-
public function allocate() {
45
-
$lease = $this->getPendingLease();
46
-
47
-
$data = array(
48
-
'type' => $this->getResourceType(),
49
-
'lease' => $lease->getID(),
50
-
);
51
-
52
-
$task = PhabricatorWorker::scheduleTask('DrydockAllocatorWorker', $data);
53
-
54
-
$lease->setTaskID($task->getID());
55
-
56
-
return $lease;
57
-
}
58
-
59
-
}
+5
-7
src/applications/drydock/allocator/DrydockAllocatorWorker.php
src/applications/drydock/worker/DrydockAllocatorWorker.php
+5
-7
src/applications/drydock/allocator/DrydockAllocatorWorker.php
src/applications/drydock/worker/DrydockAllocatorWorker.php
···
19
19
final class DrydockAllocatorWorker extends PhabricatorWorker {
20
20
21
21
protected function doWork() {
22
-
$data = $this->getTaskData();
22
+
$lease_id = $this->getTaskData();
23
23
24
-
$lease = id(new DrydockLease())->loadOneWhere(
25
-
'id = %d',
26
-
$data['lease']);
24
+
$lease = id(new DrydockLease())->load($lease_id);
27
25
if (!$lease) {
28
26
return;
29
27
}
30
28
31
-
$type = $data['type'];
29
+
$type = $lease->getResourceType();
32
30
33
31
$candidates = id(new DrydockResource())->loadAllWhere(
34
32
'type = %s AND status = %s',
35
-
$type,
33
+
$lease->getResourceType(),
36
34
DrydockResourceStatus::STATUS_OPEN);
37
35
38
36
if ($candidates) {
···
64
62
shuffle($blueprints);
65
63
66
64
$blueprint = head($blueprints);
67
-
$resource = $blueprint->allocateResource();
65
+
$resource = $blueprint->allocateResource($lease);
68
66
}
69
67
70
68
$blueprint = $resource->getBlueprint();
+8
-10
src/applications/drydock/blueprint/DrydockBlueprint.php
+8
-10
src/applications/drydock/blueprint/DrydockBlueprint.php
···
33
33
return;
34
34
}
35
35
36
-
protected function getAllocator($type) {
37
-
$allocator = new DrydockAllocator();
38
-
$allocator->setResourceType($type);
39
-
40
-
return $allocator;
41
-
}
42
-
43
36
final public function acquireLease(
44
37
DrydockResource $resource,
45
38
DrydockLease $lease) {
···
101
94
return false;
102
95
}
103
96
104
-
protected function executeAllocateResource() {
97
+
protected function executeAllocateResource(DrydockLease $lease) {
105
98
throw new Exception("This blueprint can not allocate resources!");
106
99
}
107
100
108
-
final public function allocateResource() {
101
+
final public function allocateResource(DrydockLease $lease) {
102
+
$this->activeLease = $lease;
103
+
$this->activeResource = null;
104
+
105
+
$this->log('Allocating Resource');
106
+
109
107
try {
110
-
$resource = $this->executeAllocateResource();
108
+
$resource = $this->executeAllocateResource($lease);
111
109
} catch (Exception $ex) {
112
110
$this->logException($ex);
113
111
$this->activeResource = null;
+2
-2
src/applications/drydock/blueprint/DrydockEC2HostBlueprint.php
+2
-2
src/applications/drydock/blueprint/DrydockEC2HostBlueprint.php
···
22
22
return true;
23
23
}
24
24
25
-
public function executeAllocateResource() {
25
+
public function executeAllocateResource(DrydockLease $lease) {
26
26
$resource = $this->newResourceTemplate('EC2 Host');
27
27
28
28
$resource->setStatus(DrydockResourceStatus::STATUS_ALLOCATING);
···
41
41
42
42
$instance_id = (string)$xml->instancesSet[0]->item[0]->instanceId[0];
43
43
44
-
$this->log('Started Instance: {$instance_id}');
44
+
$this->log("Started Instance: {$instance_id}");
45
45
$resource->setAttribute('instance.id', $instance_id);
46
46
$resource->save();
47
47
+4
-3
src/applications/drydock/blueprint/application/DrydockPhabricatorApplicationBlueprint.php
+4
-3
src/applications/drydock/blueprint/application/DrydockPhabricatorApplicationBlueprint.php
···
27
27
return true;
28
28
}
29
29
30
-
public function executeAllocateResource() {
30
+
public function executeAllocateResource(DrydockLease $lease) {
31
31
32
32
$resource = $this->newResourceTemplate('Phabricator');
33
33
34
34
$resource->setStatus(DrydockResourceStatus::STATUS_ALLOCATING);
35
35
$resource->save();
36
36
37
-
$allocator = $this->getAllocator('host');
38
-
$host = $allocator->allocate();
37
+
$host = id(new DrydockLease())
38
+
->setResourceType('host')
39
+
->queueForActivation();
39
40
40
41
$cmd = $host->waitUntilActive()->getInterface('command');
41
42
+1
-1
src/applications/drydock/blueprint/webroot/DrydockApacheWebrootBlueprint.php
+1
-1
src/applications/drydock/blueprint/webroot/DrydockApacheWebrootBlueprint.php
+22
-12
src/applications/drydock/management/DrydockManagementLeaseWorkflow.php
+22
-12
src/applications/drydock/management/DrydockManagementLeaseWorkflow.php
···
31
31
'help' => 'Resource type.',
32
32
),
33
33
array(
34
-
'name' => 'spec',
34
+
'name' => 'attributes',
35
35
'param' => 'name=value,...',
36
36
'help' => 'Resource specficiation.',
37
37
),
···
47
47
"Specify a resource type with `--type`.");
48
48
}
49
49
50
-
$spec = $args->getArg('spec');
51
-
if ($spec) {
50
+
$attributes = $args->getArg('attributes');
51
+
if ($attributes) {
52
52
$options = new PhutilSimpleOptions();
53
-
$spec = $options->parse($spec);
53
+
$attributes = $options->parse($attributes);
54
54
}
55
55
56
-
$allocator = new DrydockAllocator();
57
-
$allocator->setResourceType($resource_type);
58
-
if ($spec) {
59
-
// TODO: Shove this in there.
56
+
$lease = new DrydockLease();
57
+
$lease->setResourceType($resource_type);
58
+
if ($attributes) {
59
+
$lease->setAttributes($attributes);
60
60
}
61
-
62
-
$lease = $allocator->allocate();
61
+
$lease->queueForActivation();
63
62
64
63
$root = dirname(phutil_get_library_root('phabricator'));
65
64
$wait = new ExecFuture(
···
67
66
$root.'/scripts/drydock/drydock_control.php',
68
67
$lease->getID());
69
68
69
+
$cursor = 0;
70
70
foreach (Futures(array($wait))->setUpdateInterval(1) as $key => $future) {
71
71
if ($future) {
72
72
$future->resolvex();
73
73
break;
74
74
}
75
75
76
-
// TODO: Pull logs.
77
-
$console->writeErr("Working...\n");
76
+
$logs = id(new DrydockLogQuery())
77
+
->withLeaseIDs(array($lease->getID()))
78
+
->withAfterID($cursor)
79
+
->setOrder(DrydockLogQuery::ORDER_ID)
80
+
->execute();
81
+
82
+
if ($logs) {
83
+
foreach ($logs as $log) {
84
+
$console->writeErr("%s\n", $log->getMessage());
85
+
}
86
+
$cursor = max(mpull($logs, 'getID'));
87
+
}
78
88
}
79
89
80
90
$console->writeOut("Acquired Lease %s\n", $lease->getID());
+35
-10
src/applications/drydock/query/DrydockLogQuery.php
+35
-10
src/applications/drydock/query/DrydockLogQuery.php
···
18
18
19
19
final class DrydockLogQuery extends PhabricatorOffsetPagedQuery {
20
20
21
+
const ORDER_EPOCH = 'order-epoch';
22
+
const ORDER_ID = 'order-id';
23
+
21
24
private $resourceIDs;
22
25
private $leaseIDs;
26
+
private $afterID;
27
+
private $order = self::ORDER_EPOCH;
23
28
24
29
public function withResourceIDs(array $ids) {
25
30
$this->resourceIDs = $ids;
···
31
36
return $this;
32
37
}
33
38
39
+
public function setOrder($order) {
40
+
$this->order = $order;
41
+
return $this;
42
+
}
43
+
44
+
public function withAfterID($id) {
45
+
$this->afterID = $id;
46
+
return $this;
47
+
}
48
+
34
49
public function execute() {
35
50
$table = new DrydockLog();
36
51
$conn_r = $table->establishConnection('r');
37
52
38
-
$where = $this->buildWhereClause($conn_r);
39
-
$order = $this->buildOrderClause($conn_r);
40
-
$limit = $this->buildLimitClause($conn_r);
41
-
42
53
$data = queryfx_all(
43
54
$conn_r,
44
55
'SELECT log.* FROM %T log %Q %Q %Q',
45
56
$table->getTableName(),
46
-
$where,
47
-
$order,
48
-
$limit);
57
+
$this->buildWhereClause($conn_r),
58
+
$this->buildOrderClause($conn_r),
59
+
$this->buildLimitClause($conn_r));
49
60
50
61
return $table->loadAllFromArray($data);
51
62
}
52
63
53
-
private function buildWhereClause($conn_r) {
64
+
private function buildWhereClause(AphrontDatabaseConnection $conn_r) {
54
65
$where = array();
55
66
56
67
if ($this->resourceIDs) {
···
65
76
$conn_r,
66
77
'leaseID IN (%Ld)',
67
78
$this->leaseIDs);
79
+
}
80
+
81
+
if ($this->afterID) {
82
+
$where[] = qsprintf(
83
+
$conn_r,
84
+
'id > %d',
85
+
$this->afterID);
68
86
}
69
87
70
88
return $this->formatWhereClause($where);
71
89
}
72
90
73
-
private function buildOrderClause($conn_r) {
74
-
return 'ORDER BY log.epoch DESC';
91
+
private function buildOrderClause(AphrontDatabaseConnection $conn_r) {
92
+
switch ($this->order) {
93
+
case self::ORDER_EPOCH:
94
+
return 'ORDER BY log.epoch DESC';
95
+
case self::ORDER_ID:
96
+
return 'ORDER BY id ASC';
97
+
default:
98
+
throw new Exception("Unknown order '{$this->order}'!");
99
+
}
75
100
}
76
101
77
102
}
+31
-3
src/applications/drydock/storage/DrydockLease.php
+31
-3
src/applications/drydock/storage/DrydockLease.php
···
20
20
21
21
protected $phid;
22
22
protected $resourceID;
23
+
protected $resourceType;
23
24
protected $until;
24
25
protected $ownerPHID;
25
26
protected $attributes = array();
26
-
protected $status;
27
+
protected $status = DrydockLeaseStatus::STATUS_PENDING;
27
28
protected $taskID;
28
29
29
30
private $resource;
···
76
77
$this->getResourceID());
77
78
}
78
79
79
-
public function release() {
80
+
public function queueForActivation() {
81
+
if ($this->getID()) {
82
+
throw new Exception(
83
+
"Only new leases may be queued for activation!");
84
+
}
85
+
86
+
$this->setStatus(DrydockLeaseStatus::STATUS_PENDING);
87
+
$this->save();
88
+
89
+
// NOTE: Prevent a race where some eager worker quickly grabs the task
90
+
// before we can save the Task ID.
91
+
92
+
$this->openTransaction();
93
+
$this->beginReadLocking();
94
+
95
+
$this->reload();
96
+
97
+
$task = PhabricatorWorker::scheduleTask(
98
+
'DrydockAllocatorWorker',
99
+
$this->getID());
100
+
101
+
$this->setTaskID($task->getID());
102
+
$this->save();
103
+
104
+
$this->endReadLocking();
105
+
$this->saveTransaction();
80
106
81
-
// TODO: Insert a cleanup task into the taskmaster queue.
107
+
return $this;
108
+
}
82
109
110
+
public function release() {
83
111
$this->setStatus(DrydockLeaseStatus::STATUS_RELEASED);
84
112
$this->save();
85
113
+4
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php
+4
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php