@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Undumb the Drydock resource allocator pipeline
Summary:
This was the major goal of D3859/D3855, and to a lesser degree D3854/D3852.
As Drydock is allocating a resource, it may need to allocate other resources first. For example, if it's allocating a working copy, it may need to allocate a host first.
Currently, we have the process basically queue up the allocation (insert a task into the queue) and sleep() until it finishes. This is problematic for a bunch of reasons, but the major one is that if allocation takes more resources (host, port, machine, DNS) than you have daemons, they could all end up sleeping and waiting for some other daemon to do their work. This is really stupid. Even if you only take up some of them, you're spending slots sleeping when you could be doing useful work.
To partially get around this and make the CLI experience less dumb, there's this goofy `synchronous` flag that gets passed around everywhere and pushes the workflow through a pile of special cases. Basically the `synchronous` flag causes us to do everything in-process. But this is dumb too because we'd rather do things in parallel if we can, and we have to have a lot of special case code to make it work at all.
Get rid of all of this. Instead of sleep()ing, try to work on the tasks that need to be worked on. If another daemon grabbed them already that's fine, but in the worst case we just gracefully degrade and do everything in process. So we get the best of both worlds: if we have parallelizable tasks and free daemons, things will execute in parallel. If we have nonparallelizable tasks or no free daemons, things will execute in process.
Test Plan: Ran `drydock_control.php --trace` and saw it perform cascading allocations without sleeping or special casing.
Reviewers: btrahan
Reviewed By: btrahan
CC: aran
Maniphest Tasks: T2015
Differential Revision: https://secure.phabricator.com/D3861
+102
-48
+1
resources/sql/patches/drydocktaskid.sql
+1
resources/sql/patches/drydocktaskid.sql
···
1
+
ALTER TABLE {$NAMESPACE}_drydock.drydock_lease ADD taskID INT UNSIGNED;
+5
-4
scripts/drydock/drydock_control.php
+5
-4
scripts/drydock/drydock_control.php
···
20
20
$root = dirname(dirname(dirname(__FILE__)));
21
21
require_once $root.'/scripts/__init_script__.php';
22
22
23
-
PhutilServiceProfiler::installEchoListener();
23
+
$args = new PhutilArgumentParser($argv);
24
+
$args->parseStandardArguments();
25
+
$args->parse(array());
24
26
25
27
$allocator = new DrydockAllocator();
26
-
$allocator->makeSynchronous();
27
28
$allocator->setResourceType('webroot');
28
29
$lease = $allocator->allocate();
29
30
···
46
47
$lease->release();
47
48
48
49
49
-
//$i_http = $lease->getInterface('httpd');
50
-
//echo $i_http->getURI('/index.html')."\n";
50
+
// $i_http = $lease->getInterface('httpd');
51
+
// echo $i_http->getURI('/index.html')."\n";
+3
-12
src/applications/drydock/allocator/DrydockAllocator.php
+3
-12
src/applications/drydock/allocator/DrydockAllocator.php
···
20
20
21
21
private $resourceType;
22
22
private $lease;
23
-
private $synchronous;
24
23
25
24
public function setResourceType($resource_type) {
26
25
$this->resourceType = $resource_type;
···
29
28
30
29
public function getResourceType() {
31
30
return $this->resourceType;
32
-
}
33
-
34
-
public function makeSynchronous() {
35
-
$this->synchronous = true;
36
31
}
37
32
38
33
public function getPendingLease() {
···
54
49
'lease' => $lease->getID(),
55
50
);
56
51
57
-
if ($this->synchronous) {
58
-
$data['synchronous'] = true;
59
-
$worker = new DrydockAllocatorWorker($data);
60
-
$worker->executeTask();
61
-
} else {
62
-
PhabricatorWorker::scheduleTask('DrydockAllocatorWorker', $data);
63
-
}
52
+
$task = PhabricatorWorker::scheduleTask('DrydockAllocatorWorker', $data);
53
+
54
+
$lease->setTaskID($task->getID());
64
55
65
56
return $lease;
66
57
}
-5
src/applications/drydock/allocator/DrydockAllocatorWorker.php
-5
src/applications/drydock/allocator/DrydockAllocatorWorker.php
-8
src/applications/drydock/blueprint/DrydockBlueprint.php
-8
src/applications/drydock/blueprint/DrydockBlueprint.php
···
20
20
21
21
private $activeLease;
22
22
private $activeResource;
23
-
private $synchronous;
24
-
25
-
final public function makeSynchronous() {
26
-
$this->synchronous = true;
27
-
}
28
23
29
24
abstract public function getType();
30
25
abstract public function getInterface(
···
40
35
41
36
protected function getAllocator($type) {
42
37
$allocator = new DrydockAllocator();
43
-
if ($this->synchronous) {
44
-
$allocator->makeSynchronous();
45
-
}
46
38
$allocator->setResourceType($type);
47
39
48
40
return $allocator;
+33
-14
src/applications/drydock/storage/DrydockLease.php
+33
-14
src/applications/drydock/storage/DrydockLease.php
···
24
24
protected $ownerPHID;
25
25
protected $attributes = array();
26
26
protected $status;
27
+
protected $taskID;
27
28
28
29
private $resource;
29
30
···
95
96
}
96
97
}
97
98
98
-
public function waitUntilActive() {
99
-
$this->reload();
99
+
public static function waitForLeases(array $leases) {
100
+
assert_instances_of($leases, 'DrydockLease');
101
+
102
+
$task_ids = array_filter(mpull($leases, 'getTaskID'));
103
+
PhabricatorWorker::waitForTasks($task_ids);
104
+
105
+
$unresolved = $leases;
100
106
while (true) {
101
-
switch ($this->status) {
102
-
case DrydockLeaseStatus::STATUS_ACTIVE:
103
-
break 2;
104
-
case DrydockLeaseStatus::STATUS_RELEASED:
105
-
case DrydockLeaseStatus::STATUS_EXPIRED:
106
-
case DrydockLeaseStatus::STATUS_BROKEN:
107
-
throw new Exception("Lease will never become active!");
108
-
case DrydockLeaseStatus::STATUS_PENDING:
109
-
break;
107
+
foreach ($unresolved as $key => $lease) {
108
+
$lease->reload();
109
+
switch ($lease->getStatus()) {
110
+
case DrydockLeaseStatus::STATUS_ACTIVE:
111
+
unset($unresolved[$key]);
112
+
break;
113
+
case DrydockLeaseStatus::STATUS_RELEASED:
114
+
case DrydockLeaseStatus::STATUS_EXPIRED:
115
+
case DrydockLeaseStatus::STATUS_BROKEN:
116
+
throw new Exception("Lease will never become active!");
117
+
case DrydockLeaseStatus::STATUS_PENDING:
118
+
break;
119
+
}
110
120
}
111
-
sleep(2);
112
-
$this->reload();
121
+
122
+
if ($unresolved) {
123
+
sleep(1);
124
+
} else {
125
+
break;
126
+
}
113
127
}
114
128
115
-
$this->attachResource($this->loadResource());
129
+
foreach ($leases as $lease) {
130
+
$lease->attachResource($lease->loadResource());
131
+
}
132
+
}
116
133
134
+
public function waitUntilActive() {
135
+
self::waitForLeases(array($this));
117
136
return $this;
118
137
}
119
138
+43
src/infrastructure/daemon/workers/PhabricatorWorker.php
+43
src/infrastructure/daemon/workers/PhabricatorWorker.php
···
107
107
->save();
108
108
}
109
109
110
+
111
+
/**
112
+
* Wait for tasks to complete. If tasks are not leased by other workers, they
113
+
* will be executed in this process while waiting.
114
+
*
115
+
* @param list<int> List of queued task IDs to wait for.
116
+
* @return void
117
+
*/
118
+
final public static function waitForTasks(array $task_ids) {
119
+
$task_table = new PhabricatorWorkerActiveTask();
120
+
121
+
$waiting = array_combine($task_ids, $task_ids);
122
+
while ($waiting) {
123
+
$conn_w = $task_table->establishConnection('w');
124
+
125
+
// Check if any of the tasks we're waiting on are still queued. If they
126
+
// are not, we're done waiting.
127
+
$row = queryfx_one(
128
+
$conn_w,
129
+
'SELECT COUNT(*) N FROM %T WHERE id IN (%Ld)',
130
+
$task_table->getTableName(),
131
+
$waiting);
132
+
if (!$row['N']) {
133
+
// Nothing is queued anymore. Stop waiting.
134
+
break;
135
+
}
136
+
137
+
$tasks = id(new PhabricatorWorkerLeaseQuery())
138
+
->withIDs($waiting)
139
+
->setLimit(1)
140
+
->execute();
141
+
142
+
if (!$tasks) {
143
+
// We were not successful in leasing anything. Sleep for a bit and
144
+
// see if we have better luck later.
145
+
sleep(1);
146
+
continue;
147
+
}
148
+
149
+
$task = head($tasks)->executeTask();
150
+
}
151
+
}
152
+
110
153
}
+13
-5
src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php
+13
-5
src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php
···
25
25
26
26
const PHASE_UNLEASED = 'unleased';
27
27
const PHASE_EXPIRED = 'expired';
28
+
const PHASE_SELECT = 'select';
28
29
29
30
const DEFAULT_LEASE_DURATION = 60; // Seconds
30
31
···
66
67
foreach ($phases as $phase) {
67
68
queryfx(
68
69
$conn_w,
69
-
'UPDATE %T SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + %d
70
+
'UPDATE %T task
71
+
SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + %d
70
72
%Q %Q %Q',
71
73
$task_table->getTableName(),
72
74
$lease_ownership_name,
···
90
92
'SELECT task.*, taskdata.data _taskData, UNIX_TIMESTAMP() _serverTime
91
93
FROM %T task LEFT JOIN %T taskdata
92
94
ON taskdata.id = task.dataID
93
-
WHERE leaseOwner = %s AND leaseExpires > UNIX_TIMESTAMP()
94
-
%Q %Q',
95
+
%Q %Q %Q',
95
96
$task_table->getTableName(),
96
97
$taskdata_table->getTableName(),
97
-
$lease_ownership_name,
98
+
$this->buildWhereClause($conn_w, self::PHASE_SELECT),
98
99
$this->buildOrderClause($conn_w),
99
100
$this->buildLimitClause($conn_w, $limit));
100
101
···
124
125
case self::PHASE_EXPIRED:
125
126
$where[] = 'leaseExpires < UNIX_TIMESTAMP()';
126
127
break;
128
+
case self::PHASE_SELECT:
129
+
$where[] = qsprintf(
130
+
$conn_w,
131
+
'leaseOwner = %s',
132
+
$this->getLeaseOwnershipName());
133
+
$where[] = 'leaseExpires > UNIX_TIMESTAMP()';
134
+
break;
127
135
default:
128
136
throw new Exception("Unknown phase '{$phase}'!");
129
137
}
···
131
139
if ($this->ids) {
132
140
$where[] = qsprintf(
133
141
$conn_w,
134
-
'id IN (%Ld)',
142
+
'task.id IN (%Ld)',
135
143
$this->ids);
136
144
}
137
145
+4
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php
+4
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php