@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Make PullLocal smart about which repositories it should pull
Summary:
Ref T10756. When repositories are properly configured for the cluster (which is hard to set up today), be smart about which repositories are expected to exist on the current host, and only pull them.
This generally allows daemons to pretty much do the right thing no matter how many copies are running, although there may still be some lock contention issues that need to be sorted out.
Test Plan: {F1214483}
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T10756
Differential Revision: https://secure.phabricator.com/D15682
+141
-5
+3
-3
src/applications/almanac/controller/AlmanacDeviceViewController.php
+3
-3
src/applications/almanac/controller/AlmanacDeviceViewController.php
···
117
117
->setCanEdit($can_edit);
118
118
119
119
$header = id(new PHUIHeaderView())
120
-
->setHeader(pht('DEVICE INTERFACES'))
120
+
->setHeader(pht('Device Interfaces'))
121
121
->addActionLink(
122
122
id(new PHUIButtonView())
123
123
->setTag('a')
···
167
167
$upload_uri = '/auth/sshkey/upload/?objectPHID='.$device_phid;
168
168
169
169
$header = id(new PHUIHeaderView())
170
-
->setHeader(pht('SSH PUBLIC KEYS'))
170
+
->setHeader(pht('SSH Public Keys'))
171
171
->addActionLink(
172
172
id(new PHUIButtonView())
173
173
->setTag('a')
···
238
238
));
239
239
240
240
return id(new PHUIObjectBoxView())
241
-
->setHeaderText(pht('BOUND SERVICES'))
241
+
->setHeaderText(pht('Bound Services'))
242
242
->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
243
243
->setTable($table);
244
244
}
+29
src/applications/almanac/util/AlmanacKeys.php
+29
src/applications/almanac/util/AlmanacKeys.php
···
19
19
return null;
20
20
}
21
21
22
+
public static function getLiveDevice() {
23
+
$device_id = self::getDeviceID();
24
+
if (!$device_id) {
25
+
return null;
26
+
}
27
+
28
+
$cache = PhabricatorCaches::getRequestCache();
29
+
$cache_key = 'almanac.device.self';
30
+
31
+
$device = $cache->getKey($cache_key);
32
+
if (!$device) {
33
+
$viewer = PhabricatorUser::getOmnipotentUser();
34
+
$device = id(new AlmanacDeviceQuery())
35
+
->setViewer($viewer)
36
+
->withNames(array($device_id))
37
+
->executeOne();
38
+
if (!$device) {
39
+
throw new Exception(
40
+
pht(
41
+
'This host has device ID "%s", but there is no corresponding '.
42
+
'device record in Almanac.',
43
+
$device_id));
44
+
}
45
+
$cache->setKey($cache_key, $device);
46
+
}
47
+
48
+
return $device;
49
+
}
50
+
22
51
}
+109
-2
src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
+109
-2
src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
···
74
74
75
75
while (!$this->shouldExit()) {
76
76
PhabricatorCaches::destroyRequestCache();
77
-
$pullable = $this->loadPullableRepositories($include, $exclude);
77
+
$device = AlmanacKeys::getLiveDevice();
78
+
79
+
$pullable = $this->loadPullableRepositories($include, $exclude, $device);
78
80
79
81
// If any repositories have the NEEDS_UPDATE flag set, pull them
80
82
// as soon as possible.
···
297
299
/**
298
300
* @task pull
299
301
*/
300
-
private function loadPullableRepositories(array $include, array $exclude) {
302
+
private function loadPullableRepositories(
303
+
array $include,
304
+
array $exclude,
305
+
AlmanacDevice $device = null) {
306
+
301
307
$query = id(new PhabricatorRepositoryQuery())
302
308
->setViewer($this->getViewer());
303
309
···
345
351
foreach ($repositories as $key => $repository) {
346
352
if (!$repository->isTracked()) {
347
353
unset($repositories[$key]);
354
+
}
355
+
}
356
+
357
+
$service_phids = array();
358
+
foreach ($repositories as $key => $repository) {
359
+
$service_phid = $repository->getAlmanacServicePHID();
360
+
361
+
// If the repository is bound to a service but this host is not a
362
+
// recognized device, or vice versa, don't pull the repository.
363
+
$is_cluster_repo = (bool)$service_phid;
364
+
$is_cluster_device = (bool)$device;
365
+
if ($is_cluster_repo != $is_cluster_device) {
366
+
if ($is_cluster_device) {
367
+
$this->log(
368
+
pht(
369
+
'Repository "%s" is not a cluster repository, but the current '.
370
+
'host is a cluster device ("%s"), so the repository will not '.
371
+
'be updated on this host.',
372
+
$repository->getDisplayName(),
373
+
$device->getName()));
374
+
} else {
375
+
$this->log(
376
+
pht(
377
+
'Repository "%s" is a cluster repository, but the current '.
378
+
'host is not a cluster device (it has no device ID), so the '.
379
+
'repository will not be updated on this host.',
380
+
$repository->getDisplayName()));
381
+
}
382
+
unset($repositories[$key]);
383
+
continue;
384
+
}
385
+
386
+
if ($service_phid) {
387
+
$service_phids[] = $service_phid;
388
+
}
389
+
}
390
+
391
+
if ($device) {
392
+
$device_phid = $device->getPHID();
393
+
394
+
if ($service_phids) {
395
+
// We could include `withDevicePHIDs()` here to pull a smaller result
396
+
// set, but we can provide more helpful diagnostic messages below if
397
+
// we fetch a little more data.
398
+
$services = id(new AlmanacServiceQuery())
399
+
->setViewer($this->getViewer())
400
+
->withPHIDs($service_phids)
401
+
->needBindings(true)
402
+
->execute();
403
+
$services = mpull($services, null, 'getPHID');
404
+
} else {
405
+
$services = array();
406
+
}
407
+
408
+
foreach ($repositories as $key => $repository) {
409
+
$service_phid = $repository->getAlmanacServicePHID();
410
+
411
+
$service = idx($services, $service_phid);
412
+
if (!$service) {
413
+
$this->log(
414
+
pht(
415
+
'Repository "%s" is on cluster service "%s", but that service '.
416
+
'could not be loaded, so the repository will not be updated '.
417
+
'on this host.',
418
+
$repository->getDisplayName(),
419
+
$service_phid));
420
+
unset($repositories[$key]);
421
+
continue;
422
+
}
423
+
424
+
$bindings = $service->getBindings();
425
+
$bindings = mpull($bindings, null, 'getDevicePHID');
426
+
$binding = idx($bindings, $device_phid);
427
+
if (!$binding) {
428
+
$this->log(
429
+
pht(
430
+
'Repository "%s" is on cluster service "%s", but that service '.
431
+
'is not bound to this device ("%s"), so the repository will '.
432
+
'not be updated on this host.',
433
+
$repository->getDisplayName(),
434
+
$service->getName(),
435
+
$device->getName()));
436
+
unset($repositories[$key]);
437
+
continue;
438
+
}
439
+
440
+
if ($binding->getIsDisabled()) {
441
+
$this->log(
442
+
pht(
443
+
'Repository "%s" is on cluster service "%s", but the binding '.
444
+
'between that service and this device ("%s") is disabled, so '.
445
+
'the not be updated on this host.',
446
+
$repository->getDisplayName(),
447
+
$service->getName(),
448
+
$device->getName()));
449
+
unset($repositories[$key]);
450
+
continue;
451
+
}
452
+
453
+
// We have a valid service that is actively bound to the current host
454
+
// device, so we're good to go.
348
455
}
349
456
}
350
457