@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Make repository daemon locks more granular and forgiving
Summary:
Ref T4292. Currently, we hold one big lock around the whole `bin/repository update` workflow.
When running multiple daemons on different hosts, this lock can end up being contentious. In particular, we'll hold it during `git fetch` on every host globally, even though it's only useful to hold it locally per-device (that is, it's fine/good/expected if `repo001` and `repo002` happen to be fetching from a repository they are observing at the same time).
Instead, split it into two locks:
- One lock is scoped to the current device, and held during pull (usually `git fetch`). This just keeps multiple daemons accidentally running on the same host from making a mess when trying to initialize or update a working copy.
- One lock is scoped globally, and held during discovery. This makes sure daemons on different hosts don't step on each other when updating the database.
If we fail to acquire either lock, assume some other process is legitimately doing the work and bail more quietly instead of fataling. In approximately 100% of cases where users have hit this lock contention, that was the case: some other daemon was running somewhere doing the work and the error didn't actually represent an issue.
If there's an actual problem, we still raise a diagnostically useful message if you run `bin/repository update` manually, so there are still tools to figure out that something is hung or whatever.
Test Plan:
- Ran `bin/repository update`, `pull`, `discover`.
- Added `sleep(5)`, forced processes to contend, got lock exceptions and graceful exit with diagnostic message.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T4292
Differential Revision: https://secure.phabricator.com/D15903
+119
-50
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
622
622
'DiffusionController' => 'applications/diffusion/controller/DiffusionController.php',
623
623
'DiffusionCreateCommentConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionCreateCommentConduitAPIMethod.php',
624
624
'DiffusionCreateRepositoriesCapability' => 'applications/diffusion/capability/DiffusionCreateRepositoriesCapability.php',
625
+
'DiffusionDaemonLockException' => 'applications/diffusion/exception/DiffusionDaemonLockException.php',
625
626
'DiffusionDefaultEditCapability' => 'applications/diffusion/capability/DiffusionDefaultEditCapability.php',
626
627
'DiffusionDefaultPushCapability' => 'applications/diffusion/capability/DiffusionDefaultPushCapability.php',
627
628
'DiffusionDefaultViewCapability' => 'applications/diffusion/capability/DiffusionDefaultViewCapability.php',
···
4845
4846
'DiffusionController' => 'PhabricatorController',
4846
4847
'DiffusionCreateCommentConduitAPIMethod' => 'DiffusionConduitAPIMethod',
4847
4848
'DiffusionCreateRepositoriesCapability' => 'PhabricatorPolicyCapability',
4849
+
'DiffusionDaemonLockException' => 'Exception',
4848
4850
'DiffusionDefaultEditCapability' => 'PhabricatorPolicyCapability',
4849
4851
'DiffusionDefaultPushCapability' => 'PhabricatorPolicyCapability',
4850
4852
'DiffusionDefaultViewCapability' => 'PhabricatorPolicyCapability',
+8
src/applications/almanac/util/AlmanacKeys.php
+8
src/applications/almanac/util/AlmanacKeys.php
···
10
10
}
11
11
12
12
public static function getDeviceID() {
13
+
// While running unit tests, ignore any configured device identity.
14
+
try {
15
+
PhabricatorTestCase::assertExecutingUnitTests();
16
+
return null;
17
+
} catch (Exception $ex) {
18
+
// Continue normally.
19
+
}
20
+
13
21
$device_id_path = self::getKeyPath('device.id');
14
22
15
23
if (Filesystem::pathExists($device_id_path)) {
+3
src/applications/diffusion/exception/DiffusionDaemonLockException.php
+3
src/applications/diffusion/exception/DiffusionDaemonLockException.php
+27
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
+27
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
···
37
37
public function discoverCommits() {
38
38
$repository = $this->getRepository();
39
39
40
+
$lock = $this->newRepositoryLock($repository, 'repo.look', false);
41
+
42
+
try {
43
+
$lock->lock();
44
+
} catch (PhutilLockException $ex) {
45
+
throw new DiffusionDaemonLockException(
46
+
pht(
47
+
'Another process is currently discovering repository "%s", '.
48
+
'skipping discovery.',
49
+
$repository->getDisplayName()));
50
+
}
51
+
52
+
try {
53
+
$result = $this->discoverCommitsWithLock();
54
+
} catch (Exception $ex) {
55
+
$lock->unlock();
56
+
throw $ex;
57
+
}
58
+
59
+
$lock->unlock();
60
+
61
+
return $result;
62
+
}
63
+
64
+
private function discoverCommitsWithLock() {
65
+
$repository = $this->getRepository();
66
+
40
67
$vcs = $repository->getVersionControlSystem();
41
68
switch ($vcs) {
42
69
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
+21
src/applications/repository/engine/PhabricatorRepositoryEngine.php
+21
src/applications/repository/engine/PhabricatorRepositoryEngine.php
···
51
51
return PhabricatorUser::getOmnipotentUser();
52
52
}
53
53
54
+
protected function newRepositoryLock(
55
+
PhabricatorRepository $repository,
56
+
$lock_key,
57
+
$lock_device_only) {
58
+
59
+
$lock_parts = array();
60
+
$lock_parts[] = $lock_key;
61
+
$lock_parts[] = $repository->getID();
62
+
63
+
if ($lock_device_only) {
64
+
$device = AlmanacKeys::getLiveDevice();
65
+
if ($device) {
66
+
$lock_parts[] = $device->getID();
67
+
}
68
+
}
69
+
70
+
$lock_name = implode(':', $lock_parts);
71
+
return PhabricatorGlobalLock::newLock($lock_name);
72
+
}
73
+
74
+
54
75
/**
55
76
* Verify that the "origin" remote exists, and points at the correct URI.
56
77
*
+27
src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
+27
src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
···
23
23
24
24
public function pullRepository() {
25
25
$repository = $this->getRepository();
26
+
27
+
$lock = $this->newRepositoryLock($repository, 'repo.pull', true);
28
+
29
+
try {
30
+
$lock->lock();
31
+
} catch (PhutilLockException $ex) {
32
+
throw new DiffusionDaemonLockException(
33
+
pht(
34
+
'Another process is currently updating repository "%s", '.
35
+
'skipping pull.',
36
+
$repository->getDisplayName()));
37
+
}
38
+
39
+
try {
40
+
$result = $this->pullRepositoryWithLock();
41
+
} catch (Exception $ex) {
42
+
$lock->unlock();
43
+
throw $ex;
44
+
}
45
+
46
+
$lock->unlock();
47
+
48
+
return $result;
49
+
}
50
+
51
+
private function pullRepositoryWithLock() {
52
+
$repository = $this->getRepository();
26
53
$viewer = PhabricatorUser::getOmnipotentUser();
27
54
28
55
$is_hg = false;
+31
-50
src/applications/repository/management/PhabricatorRepositoryManagementUpdateWorkflow.php
+31
-50
src/applications/repository/management/PhabricatorRepositoryManagementUpdateWorkflow.php
···
53
53
$repository = head($repos);
54
54
55
55
try {
56
-
$lock_name = 'repository.update:'.$repository->getID();
57
-
$lock = PhabricatorGlobalLock::newLock($lock_name);
56
+
id(new PhabricatorRepositoryPullEngine())
57
+
->setRepository($repository)
58
+
->setVerbose($this->getVerbose())
59
+
->pullRepository();
58
60
59
-
try {
60
-
$lock->lock();
61
-
} catch (PhutilLockException $ex) {
62
-
throw new PhutilProxyException(
63
-
pht(
64
-
'Another process is currently holding the update lock for '.
65
-
'repository "%s". Repositories may only be updated by one '.
66
-
'process at a time. This can happen if you are running multiple '.
67
-
'copies of the daemons. This can also happen if you manually '.
68
-
'update a repository while the daemons are also updating it '.
69
-
'(in this case, just try again in a few moments).',
70
-
$repository->getMonogram()),
71
-
$ex);
61
+
$no_discovery = $args->getArg('no-discovery');
62
+
if ($no_discovery) {
63
+
return 0;
72
64
}
73
65
74
-
try {
75
-
$no_discovery = $args->getArg('no-discovery');
66
+
// TODO: It would be nice to discover only if we pulled something, but
67
+
// this isn't totally trivial. It's slightly more complicated with
68
+
// hosted repositories, too.
76
69
77
-
id(new PhabricatorRepositoryPullEngine())
78
-
->setRepository($repository)
79
-
->setVerbose($this->getVerbose())
80
-
->pullRepository();
70
+
$repository->writeStatusMessage(
71
+
PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
72
+
null);
81
73
82
-
if ($no_discovery) {
83
-
$lock->unlock();
84
-
return;
85
-
}
74
+
$this->discoverRepository($repository);
86
75
87
-
// TODO: It would be nice to discover only if we pulled something, but
88
-
// this isn't totally trivial. It's slightly more complicated with
89
-
// hosted repositories, too.
90
-
91
-
$repository->writeStatusMessage(
92
-
PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
93
-
null);
94
-
95
-
$this->discoverRepository($repository);
96
-
97
-
$this->checkIfRepositoryIsFullyImported($repository);
76
+
$this->checkIfRepositoryIsFullyImported($repository);
98
77
99
-
$this->updateRepositoryRefs($repository);
78
+
$this->updateRepositoryRefs($repository);
100
79
101
-
$this->mirrorRepository($repository);
80
+
$this->mirrorRepository($repository);
102
81
103
-
$repository->writeStatusMessage(
104
-
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
105
-
PhabricatorRepositoryStatusMessage::CODE_OKAY);
106
-
} catch (Exception $ex) {
107
-
$lock->unlock();
108
-
throw $ex;
109
-
}
82
+
$repository->writeStatusMessage(
83
+
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
84
+
PhabricatorRepositoryStatusMessage::CODE_OKAY);
85
+
} catch (DiffusionDaemonLockException $ex) {
86
+
// If we miss a pull or discover because some other process is already
87
+
// doing the work, just bail out.
88
+
echo tsprintf(
89
+
"%s\n",
90
+
$ex->getMessage());
91
+
return 0;
110
92
} catch (Exception $ex) {
111
93
$repository->writeStatusMessage(
112
94
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
···
118
100
throw $ex;
119
101
}
120
102
121
-
$lock->unlock();
122
-
123
-
$console->writeOut(
103
+
echo tsprintf(
104
+
"%s\n",
124
105
pht(
125
-
'Updated repository **%s**.',
126
-
$repository->getMonogram())."\n");
106
+
'Updated repository "%s".',
107
+
$repository->getDisplayName()));
127
108
128
109
return 0;
129
110
}