@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Remove Drydock host resource limits and give working copies simple limits
Summary:
Ref T9252. Right now, we have very strict limits on Drydock: one lease per host, and one working copy per working copy blueprint.
These are silly and getting in the way of using "Land Revision" more widely, since we need at least one working copy for each landable repository.
For now, just remove the host limit and put a simple limit on working copies. This might need to be fancier some day (e.g., limit working copies per-host) but it is generally reasonable for the use cases of today.
Also add a `--background` flag to make testing a little easier.
(Limits are also less important nowadays than they were in the past, because pools expand slowly now and we seem to have stamped out all the "runaway train" bugs where allocators go crazy and allocate a million things.)
Test Plan:
- With a limit of 5, ran 10 concurrent builds and saw them finish after allocating 5 total resources.
- Removed limit, raised taskmaster concurrency to 128, ran thousands of builds in blocks of 128 or 256.
- Saw Drydock gradually expand the pool, allocating a few more working copies at first and a lot of working copies later.
- Got ~256 builds in ~140 seconds, which isn't a breakneck pace or anything but isn't too bad.
- This stuff seems to be mostly bottlenecked on `sbuild` throttling inbound SSH connections. I haven't tweaked it.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T9252
Differential Revision: https://secure.phabricator.com/D14334
+135
-32
+2
-13
src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php
+2
-13
src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php
···
109
109
DrydockBlueprint $blueprint,
110
110
DrydockResource $resource,
111
111
DrydockLease $lease) {
112
-
113
-
if (!DrydockSlotLock::isLockFree($this->getLeaseSlotLock($resource))) {
114
-
return false;
115
-
}
116
-
117
112
return true;
118
113
}
119
114
···
124
119
125
120
$lease
126
121
->setActivateWhenAcquired(true)
127
-
->needSlotLock($this->getLeaseSlotLock($resource))
128
122
->acquireOnResource($resource);
129
123
}
130
124
···
146
140
return;
147
141
}
148
142
149
-
private function getLeaseSlotLock(DrydockResource $resource) {
150
-
$resource_phid = $resource->getPHID();
151
-
return "almanac.host.lease({$resource_phid})";
152
-
}
153
-
154
143
public function getType() {
155
144
return 'host';
156
145
}
···
188
177
}
189
178
}
190
179
191
-
public function getFieldSpecifications() {
180
+
protected function getCustomFieldSpecifications() {
192
181
return array(
193
182
'almanacServicePHIDs' => array(
194
183
'name' => pht('Almanac Services'),
···
207
196
'credential.type' =>
208
197
PassphraseSSHPrivateKeyTextCredentialType::CREDENTIAL_TYPE,
209
198
),
210
-
) + parent::getFieldSpecifications();
199
+
);
211
200
}
212
201
213
202
private function loadServices(DrydockBlueprint $blueprint) {
+105
-4
src/applications/drydock/blueprint/DrydockBlueprintImplementation.php
+105
-4
src/applications/drydock/blueprint/DrydockBlueprintImplementation.php
···
16
16
abstract public function getDescription();
17
17
18
18
public function getFieldSpecifications() {
19
+
$fields = array();
20
+
21
+
$fields += $this->getCustomFieldSpecifications();
22
+
23
+
if ($this->shouldUseConcurrentResourceLimit()) {
24
+
$fields += array(
25
+
'allocator.limit' => array(
26
+
'name' => pht('Limit'),
27
+
'caption' => pht(
28
+
'Maximum number of resources this blueprint can have active '.
29
+
'concurrently.'),
30
+
'type' => 'int',
31
+
),
32
+
);
33
+
}
34
+
35
+
return $fields;
36
+
}
37
+
38
+
protected function getCustomFieldSpecifications() {
19
39
return array();
20
40
}
21
41
···
317
337
318
338
319
339
/**
340
+
* Does this implementation use concurrent resource limits?
341
+
*
342
+
* Implementations can override this method to opt into standard limit
343
+
* behavior, which provides a simple concurrent resource limit.
344
+
*
345
+
* @return bool True to use limits.
346
+
*/
347
+
protected function shouldUseConcurrentResourceLimit() {
348
+
return false;
349
+
}
350
+
351
+
352
+
/**
353
+
* Get the effective concurrent resource limit for this blueprint.
354
+
*
355
+
* @param DrydockBlueprint Blueprint to get the limit for.
356
+
* @return int|null Limit, or `null` for no limit.
357
+
*/
358
+
protected function getConcurrentResourceLimit(DrydockBlueprint $blueprint) {
359
+
if ($this->shouldUseConcurrentResourceLimit()) {
360
+
$limit = $blueprint->getFieldValue('allocator.limit');
361
+
$limit = (int)$limit;
362
+
if ($limit > 0) {
363
+
return $limit;
364
+
} else {
365
+
return null;
366
+
}
367
+
}
368
+
369
+
return null;
370
+
}
371
+
372
+
373
+
protected function getConcurrentResourceLimitSlotLock(
374
+
DrydockBlueprint $blueprint) {
375
+
376
+
$limit = $this->getConcurrentResourceLimit($blueprint);
377
+
if ($limit === null) {
378
+
return;
379
+
}
380
+
381
+
$blueprint_phid = $blueprint->getPHID();
382
+
383
+
// TODO: This logic shouldn't do anything awful, but is a little silly. It
384
+
// would be nice to unify the "huge limit" and "small limit" cases
385
+
// eventually but it's a little tricky.
386
+
387
+
// If the limit is huge, just pick a random slot. This is just stopping
388
+
// us from exploding if someone types a billion zillion into the box.
389
+
if ($limit > 1024) {
390
+
$slot = mt_rand(0, $limit - 1);
391
+
return "allocator({$blueprint_phid}).limit({$slot})";
392
+
}
393
+
394
+
// For reasonable limits, actually check for an available slot.
395
+
$locks = DrydockSlotLock::loadLocks($blueprint_phid);
396
+
$locks = mpull($locks, null, 'getLockKey');
397
+
398
+
$slots = range(0, $limit - 1);
399
+
shuffle($slots);
400
+
401
+
foreach ($slots as $slot) {
402
+
$slot_lock = "allocator({$blueprint_phid}).limit({$slot})";
403
+
if (empty($locks[$slot_lock])) {
404
+
return $slot_lock;
405
+
}
406
+
}
407
+
408
+
// If we found no free slot, just return whatever we checked last (which
409
+
// is just a random slot). There's a small chance we'll get lucky and the
410
+
// lock will be free by the time we try to take it, but usually we'll just
411
+
// fail to grab the lock, throw an appropriate lock exception, and get back
412
+
// on the right path to retry later.
413
+
return $slot_lock;
414
+
}
415
+
416
+
417
+
418
+
/**
320
419
* Apply standard limits on resource allocation rate.
321
420
*
322
421
* @param DrydockBlueprint The blueprint requesting an allocation.
···
329
428
// configurable by the blueprint implementation.
330
429
331
430
// Limit on total number of active resources.
332
-
$total_limit = 1;
431
+
$total_limit = $this->getConcurrentResourceLimit($blueprint);
333
432
334
433
// Always allow at least this many allocations to be in flight at once.
335
434
$min_allowed = 1;
···
358
457
359
458
// If we're at the limit on total active resources, limit additional
360
459
// allocations.
361
-
$n_total = ($n_alloc + $n_active + $n_broken + $n_released);
362
-
if ($n_total >= $total_limit) {
363
-
return true;
460
+
if ($total_limit !== null) {
461
+
$n_total = ($n_alloc + $n_active + $n_broken + $n_released);
462
+
if ($n_total >= $total_limit) {
463
+
return true;
464
+
}
364
465
}
365
466
366
467
// If the number of in-flight allocations is fewer than the minimum number
+18
-14
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
+18
-14
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
···
125
125
->setOwnerPHID($resource_phid)
126
126
->setAttribute('workingcopy.resourcePHID', $resource_phid)
127
127
->setAllowedBlueprintPHIDs($blueprint_phids);
128
-
129
-
$resource
130
-
->setAttribute('host.leasePHID', $host_lease->getPHID())
131
-
->save();
132
-
133
-
$host_lease->queueForActivation();
134
-
135
-
// TODO: Add some limits to the number of working copies we can have at
136
-
// once?
128
+
$resource->setAttribute('host.leasePHID', $host_lease->getPHID());
137
129
138
130
$map = $lease->getAttribute('repositories.map');
139
131
foreach ($map as $key => $value) {
···
143
135
'phid',
144
136
));
145
137
}
138
+
$resource->setAttribute('repositories.map', $map);
146
139
147
-
return $resource
148
-
->setAttribute('repositories.map', $map)
149
-
->allocateResource();
140
+
$slot_lock = $this->getConcurrentResourceLimitSlotLock($blueprint);
141
+
if ($slot_lock !== null) {
142
+
$resource->needSlotLock($slot_lock);
143
+
}
144
+
145
+
$resource->allocateResource();
146
+
147
+
$host_lease->queueForActivation();
148
+
149
+
return $resource;
150
150
}
151
151
152
152
public function activateResource(
···
393
393
return $lease;
394
394
}
395
395
396
-
public function getFieldSpecifications() {
396
+
protected function getCustomFieldSpecifications() {
397
397
return array(
398
398
'blueprintPHIDs' => array(
399
399
'name' => pht('Use Blueprints'),
400
400
'type' => 'blueprints',
401
401
'required' => true,
402
402
),
403
-
) + parent::getFieldSpecifications();
403
+
);
404
+
}
405
+
406
+
protected function shouldUseConcurrentResourceLimit() {
407
+
return true;
404
408
}
405
409
406
410
+10
-1
src/applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php
+10
-1
src/applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php
···
16
16
'help' => pht('ID of build plan to run.'),
17
17
),
18
18
array(
19
+
'name' => 'background',
20
+
'help' => pht(
21
+
'Submit builds into the build queue normally instead of '.
22
+
'running them in the foreground.'),
23
+
),
24
+
array(
19
25
'name' => 'buildable',
20
26
'wildcard' => true,
21
27
),
···
88
94
"\n %s\n\n",
89
95
PhabricatorEnv::getProductionURI('/B'.$buildable->getID()));
90
96
91
-
PhabricatorWorker::setRunAllTasksInProcess(true);
97
+
if (!$args->getArg('background')) {
98
+
PhabricatorWorker::setRunAllTasksInProcess(true);
99
+
}
100
+
92
101
$buildable->applyPlan($plan, array());
93
102
94
103
$console->writeOut("%s\n", pht('Done.'));