@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Separate repository updates from the pull daemon
Summary:
Ref T4605. Currently, the PullLocal daemon is responsible for two relatively distinct things:
- scheduling repository updates; and
- actually updating repositories.
Move the "actually updating" part into a new `bin/repository update` command, which basically runs the pull, discover, refs and mirror commands. This will let the parent process focus on scheduling in a more understandable way and update multiple repositories at once. It also makes it easier to debug and understand update behavior since the non-scheduling pipeline can be run separately.
Test Plan:
- Ran `update --trace` on SVN, Mercurial and Git repos.
- Ran PullLocal daemon for a while without issues.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: epriestley
Maniphest Tasks: T4605
Differential Revision: https://secure.phabricator.com/D8780
+196
-120
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
1964
1964
'PhabricatorRepositoryManagementMirrorWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMirrorWorkflow.php',
1965
1965
'PhabricatorRepositoryManagementPullWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementPullWorkflow.php',
1966
1966
'PhabricatorRepositoryManagementRefsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementRefsWorkflow.php',
1967
+
'PhabricatorRepositoryManagementUpdateWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementUpdateWorkflow.php',
1967
1968
'PhabricatorRepositoryManagementWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementWorkflow.php',
1968
1969
'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/PhabricatorRepositoryMercurialCommitChangeParserWorker.php',
1969
1970
'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/PhabricatorRepositoryMercurialCommitMessageParserWorker.php',
···
4827
4828
'PhabricatorRepositoryManagementMirrorWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
4828
4829
'PhabricatorRepositoryManagementPullWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
4829
4830
'PhabricatorRepositoryManagementRefsWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
4831
+
'PhabricatorRepositoryManagementUpdateWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
4830
4832
'PhabricatorRepositoryManagementWorkflow' => 'PhabricatorManagementWorkflow',
4831
4833
'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
4832
4834
'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
+16
-120
src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
+16
-120
src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
···
29
29
final class PhabricatorRepositoryPullLocalDaemon
30
30
extends PhabricatorDaemon {
31
31
32
-
private $discoveryEngines = array();
33
-
34
32
35
33
/* -( Pulling Repositories )----------------------------------------------- */
36
34
···
122
120
try {
123
121
$this->log("Updating repository '{$callsign}'.");
124
122
125
-
id(new PhabricatorRepositoryPullEngine())
126
-
->setRepository($repository)
127
-
->pullRepository();
128
-
129
-
if (!$no_discovery) {
130
-
// TODO: It would be nice to discover only if we pulled something,
131
-
// but this isn't totally trivial. It's slightly more complicated
132
-
// with hosted repositories, too.
133
-
134
-
$lock_name = get_class($this).':'.$callsign;
135
-
$lock = PhabricatorGlobalLock::newLock($lock_name);
136
-
$lock->lock();
123
+
$bin_dir = dirname(phutil_get_library_root('phabricator')).'/bin';
124
+
$flags = array();
125
+
if ($no_discovery) {
126
+
$flags[] = '--no-discovery';
127
+
}
137
128
138
-
try {
139
-
$repository->writeStatusMessage(
140
-
PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
141
-
null);
142
-
$this->discoverRepository($repository);
143
-
$this->updateRepositoryRefs($repository);
144
-
$this->mirrorRepository($repository);
145
-
$repository->writeStatusMessage(
146
-
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
147
-
PhabricatorRepositoryStatusMessage::CODE_OKAY);
148
-
} catch (Exception $ex) {
149
-
$repository->writeStatusMessage(
150
-
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
151
-
PhabricatorRepositoryStatusMessage::CODE_ERROR,
152
-
array(
153
-
'message' => pht(
154
-
'Error updating working copy: %s', $ex->getMessage()),
155
-
));
156
-
$lock->unlock();
157
-
throw $ex;
158
-
}
129
+
list($stdout, $stderr) = execx(
130
+
'%s/repository update %Ls -- %s',
131
+
$bin_dir,
132
+
$flags,
133
+
$callsign);
159
134
160
-
$lock->unlock();
135
+
if (strlen($stderr)) {
136
+
$stderr_msg = pht(
137
+
'Unexpected output while updating the %s repository: %s',
138
+
$callsign,
139
+
$stderr);
140
+
phlog($stderr_msg);
161
141
}
162
142
163
143
$sleep_for = $repository->getDetail('pull-frequency', $min_sleep);
164
144
$retry_after[$id] = time() + $sleep_for;
165
-
} catch (PhutilLockException $ex) {
166
-
$retry_after[$id] = time() + $min_sleep;
167
-
$this->log("Failed to acquire lock.");
168
145
} catch (Exception $ex) {
169
146
$retry_after[$id] = time() + $min_sleep;
170
147
···
222
199
}
223
200
224
201
return $repos;
225
-
}
226
-
227
-
public function discoverRepository(PhabricatorRepository $repository) {
228
-
$refs = $this->getDiscoveryEngine($repository)
229
-
->discoverCommits();
230
-
231
-
$this->checkIfRepositoryIsFullyImported($repository);
232
-
233
-
return (bool)count($refs);
234
-
}
235
-
236
-
private function mirrorRepository(PhabricatorRepository $repository) {
237
-
try {
238
-
id(new PhabricatorRepositoryMirrorEngine())
239
-
->setRepository($repository)
240
-
->pushToMirrors();
241
-
} catch (Exception $ex) {
242
-
// TODO: We should report these into the UI properly, but for
243
-
// now just complain. These errors are much less severe than
244
-
// pull errors.
245
-
$proxy = new PhutilProxyException(
246
-
pht(
247
-
'Error while pushing "%s" repository to mirrors.',
248
-
$repository->getCallsign()),
249
-
$ex);
250
-
phlog($proxy);
251
-
}
252
-
}
253
-
254
-
private function updateRepositoryRefs(PhabricatorRepository $repository) {
255
-
id(new PhabricatorRepositoryRefEngine())
256
-
->setRepository($repository)
257
-
->updateRefs();
258
-
}
259
-
260
-
private function getDiscoveryEngine(PhabricatorRepository $repository) {
261
-
$id = $repository->getID();
262
-
if (empty($this->discoveryEngines[$id])) {
263
-
$engine = id(new PhabricatorRepositoryDiscoveryEngine())
264
-
->setRepository($repository)
265
-
->setVerbose($this->getVerbose());
266
-
267
-
$this->discoveryEngines[$id] = $engine;
268
-
}
269
-
return $this->discoveryEngines[$id];
270
-
}
271
-
272
-
private function checkIfRepositoryIsFullyImported(
273
-
PhabricatorRepository $repository) {
274
-
275
-
// Check if the repository has the "Importing" flag set. We want to clear
276
-
// the flag if we can.
277
-
$importing = $repository->getDetail('importing');
278
-
if (!$importing) {
279
-
// This repository isn't marked as "Importing", so we're done.
280
-
return;
281
-
}
282
-
283
-
// Look for any commit which hasn't imported.
284
-
$unparsed_commit = queryfx_one(
285
-
$repository->establishConnection('r'),
286
-
'SELECT * FROM %T WHERE repositoryID = %d AND (importStatus & %d) != %d
287
-
LIMIT 1',
288
-
id(new PhabricatorRepositoryCommit())->getTableName(),
289
-
$repository->getID(),
290
-
PhabricatorRepositoryCommit::IMPORTED_ALL,
291
-
PhabricatorRepositoryCommit::IMPORTED_ALL);
292
-
if ($unparsed_commit) {
293
-
// We found a commit which still needs to import, so we can't clear the
294
-
// flag.
295
-
return;
296
-
}
297
-
298
-
// Clear the "importing" flag.
299
-
$repository->openTransaction();
300
-
$repository->beginReadLocking();
301
-
$repository = $repository->reload();
302
-
$repository->setDetail('importing', false);
303
-
$repository->save();
304
-
$repository->endReadLocking();
305
-
$repository->saveTransaction();
306
202
}
307
203
308
204
}
+178
src/applications/repository/management/PhabricatorRepositoryManagementUpdateWorkflow.php
+178
src/applications/repository/management/PhabricatorRepositoryManagementUpdateWorkflow.php
···
1
+
<?php
2
+
3
+
final class PhabricatorRepositoryManagementUpdateWorkflow
4
+
extends PhabricatorRepositoryManagementWorkflow {
5
+
6
+
private $verbose;
7
+
8
+
public function setVerbose($verbose) {
9
+
$this->verbose = $verbose;
10
+
return $this;
11
+
}
12
+
13
+
public function getVerbose() {
14
+
return $this->verbose;
15
+
}
16
+
17
+
public function didConstruct() {
18
+
$this
19
+
->setName('update')
20
+
->setExamples('**update** [options] __repository__')
21
+
->setSynopsis(
22
+
pht(
23
+
'Update __repository__, named by callsign. '.
24
+
'This performs the __pull__, __discover__, __ref__ and __mirror__ '.
25
+
'operations and is primarily an internal workflow.'))
26
+
->setArguments(
27
+
array(
28
+
array(
29
+
'name' => 'verbose',
30
+
'help' => 'Show additional debugging information.',
31
+
),
32
+
array(
33
+
'name' => 'no-discovery',
34
+
'help' => 'Do not perform discovery.',
35
+
),
36
+
array(
37
+
'name' => 'repos',
38
+
'wildcard' => true,
39
+
),
40
+
));
41
+
}
42
+
43
+
public function execute(PhutilArgumentParser $args) {
44
+
$this->setVerbose($args->getArg('verbose'));
45
+
46
+
$repos = $this->loadRepositories($args, 'repos');
47
+
if (count($repos) !== 1) {
48
+
throw new PhutilArgumentUsageException(
49
+
pht('Specify exactly one repository to update, by callsign.'));
50
+
}
51
+
52
+
$repository = head($repos);
53
+
$callsign = $repository->getCallsign();
54
+
55
+
$no_discovery = $args->getArg('no-discovery');
56
+
57
+
id(new PhabricatorRepositoryPullEngine())
58
+
->setRepository($repository)
59
+
->setVerbose($this->getVerbose())
60
+
->pullRepository();
61
+
62
+
if ($no_discovery) {
63
+
return;
64
+
}
65
+
66
+
// TODO: It would be nice to discover only if we pulled something, but this
67
+
// isn't totally trivial. It's slightly more complicated with hosted
68
+
// repositories, too.
69
+
70
+
$lock_name = get_class($this).':'.$callsign;
71
+
$lock = PhabricatorGlobalLock::newLock($lock_name);
72
+
73
+
$lock->lock();
74
+
75
+
try {
76
+
$repository->writeStatusMessage(
77
+
PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
78
+
null);
79
+
80
+
$this->discoverRepository($repository);
81
+
82
+
$this->checkIfRepositoryIsFullyImported($repository);
83
+
84
+
$this->updateRepositoryRefs($repository);
85
+
86
+
$this->mirrorRepository($repository);
87
+
88
+
$repository->writeStatusMessage(
89
+
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
90
+
PhabricatorRepositoryStatusMessage::CODE_OKAY);
91
+
} catch (Exception $ex) {
92
+
$repository->writeStatusMessage(
93
+
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
94
+
PhabricatorRepositoryStatusMessage::CODE_ERROR,
95
+
array(
96
+
'message' => pht(
97
+
'Error updating working copy: %s', $ex->getMessage()),
98
+
));
99
+
100
+
$lock->unlock();
101
+
throw $ex;
102
+
}
103
+
104
+
$lock->unlock();
105
+
106
+
return 0;
107
+
}
108
+
109
+
private function discoverRepository(PhabricatorRepository $repository) {
110
+
$refs = id(new PhabricatorRepositoryDiscoveryEngine())
111
+
->setRepository($repository)
112
+
->setVerbose($this->getVerbose())
113
+
->discoverCommits();
114
+
115
+
return (bool)count($refs);
116
+
}
117
+
118
+
private function mirrorRepository(PhabricatorRepository $repository) {
119
+
try {
120
+
id(new PhabricatorRepositoryMirrorEngine())
121
+
->setRepository($repository)
122
+
->pushToMirrors();
123
+
} catch (Exception $ex) {
124
+
// TODO: We should report these into the UI properly, but for now just
125
+
// complain. These errors are much less severe than pull errors.
126
+
$proxy = new PhutilProxyException(
127
+
pht(
128
+
'Error while pushing "%s" repository to mirrors.',
129
+
$repository->getCallsign()),
130
+
$ex);
131
+
phlog($proxy);
132
+
}
133
+
}
134
+
135
+
private function updateRepositoryRefs(PhabricatorRepository $repository) {
136
+
id(new PhabricatorRepositoryRefEngine())
137
+
->setRepository($repository)
138
+
->updateRefs();
139
+
}
140
+
141
+
private function checkIfRepositoryIsFullyImported(
142
+
PhabricatorRepository $repository) {
143
+
144
+
// Check if the repository has the "Importing" flag set. We want to clear
145
+
// the flag if we can.
146
+
$importing = $repository->getDetail('importing');
147
+
if (!$importing) {
148
+
// This repository isn't marked as "Importing", so we're done.
149
+
return;
150
+
}
151
+
152
+
// Look for any commit which hasn't imported.
153
+
$unparsed_commit = queryfx_one(
154
+
$repository->establishConnection('r'),
155
+
'SELECT * FROM %T WHERE repositoryID = %d AND (importStatus & %d) != %d
156
+
LIMIT 1',
157
+
id(new PhabricatorRepositoryCommit())->getTableName(),
158
+
$repository->getID(),
159
+
PhabricatorRepositoryCommit::IMPORTED_ALL,
160
+
PhabricatorRepositoryCommit::IMPORTED_ALL);
161
+
if ($unparsed_commit) {
162
+
// We found a commit which still needs to import, so we can't clear the
163
+
// flag.
164
+
return;
165
+
}
166
+
167
+
// Clear the "importing" flag.
168
+
$repository->openTransaction();
169
+
$repository->beginReadLocking();
170
+
$repository = $repository->reload();
171
+
$repository->setDetail('importing', false);
172
+
$repository->save();
173
+
$repository->endReadLocking();
174
+
$repository->saveTransaction();
175
+
}
176
+
177
+
178
+
}