@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Make PullLocal daemon more flexible and transparent about scheduling
Summary:
Ref T4605. Fixes T3466. The major change here is that we now run up to four simultaneous updates. This should ease cases where, e.g., one very slow repository was blocking other repositories. It also tends to increase load; the next diff will introduce smart backoff for cold repositories to ease this.
The rest of this is just a ton of logging so I can IRC debug these things by having users run them in `phd debug pulllocal` mode.
For T3466:
- You now have to hit four simultaneous hangs to completely block the update process.
- Importing repository updates are killed after 4 hours.
- Imported repository updates are killed after 15 minutes.
Test Plan:
- Ran `phd debug pulllocal` and observed sensible logs and behavior.
- Interrupted daemon from sleeps and processing with `diffusion.looksoon`.
- Ran with various `--not`, `--no-discovery` flags.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: epriestley
Maniphest Tasks: T3466, T4605
Differential Revision: https://secure.phabricator.com/D8785
+256
-71
+256
-71
src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
+256
-71
src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
···
23
23
* repository).
24
24
*
25
25
* @task pull Pulling Repositories
26
-
* @task git Git Implementation
27
-
* @task hg Mercurial Implementation
28
26
*/
29
27
final class PhabricatorRepositoryPullLocalDaemon
30
28
extends PhabricatorDaemon {
···
60
58
));
61
59
62
60
$no_discovery = $args->getArg('no-discovery');
63
-
$repo_names = $args->getArg('repositories');
64
-
$exclude_names = $args->getArg('not');
61
+
$include = $args->getArg('repositories');
62
+
$exclude = $args->getArg('not');
65
63
66
64
// Each repository has an individual pull frequency; after we pull it,
67
65
// wait that long to pull it again. When we start up, try to pull everything
···
69
67
$retry_after = array();
70
68
71
69
$min_sleep = 15;
70
+
$max_futures = 4;
71
+
$futures = array();
72
+
$queue = array();
72
73
73
74
while (true) {
74
-
$repositories = $this->loadRepositories($repo_names);
75
-
if ($exclude_names) {
76
-
$exclude = $this->loadRepositories($exclude_names);
77
-
$repositories = array_diff_key($repositories, $exclude);
78
-
}
79
-
80
-
// Shuffle the repositories, then re-key the array since shuffle()
81
-
// discards keys. This is mostly for startup, we'll use soft priorities
82
-
// later.
83
-
shuffle($repositories);
84
-
$repositories = mpull($repositories, null, 'getID');
75
+
$pullable = $this->loadPullableRepositories($include, $exclude);
85
76
86
77
// If any repositories have the NEEDS_UPDATE flag set, pull them
87
78
// as soon as possible.
88
79
$need_update_messages = $this->loadRepositoryUpdateMessages();
89
80
foreach ($need_update_messages as $message) {
81
+
$repo = idx($pullable, $message->getRepositoryID());
82
+
if (!$repo) {
83
+
continue;
84
+
}
85
+
86
+
$this->log(
87
+
pht(
88
+
'Got an update message for repository "%s"!',
89
+
$repo->getMonogram()));
90
+
90
91
$retry_after[$message->getRepositoryID()] = time();
91
92
}
92
93
···
95
96
// causes us to sleep for the minimum amount of time.
96
97
$retry_after = array_select_keys(
97
98
$retry_after,
98
-
array_keys($repositories));
99
+
array_keys($pullable));
99
100
100
-
// Assign soft priorities to repositories based on how frequently they
101
-
// should pull again.
102
-
asort($retry_after);
103
-
$repositories = array_select_keys(
104
-
$repositories,
105
-
array_keys($retry_after)) + $repositories;
106
101
107
-
foreach ($repositories as $id => $repository) {
108
-
$after = idx($retry_after, $id, 0);
109
-
if ($after > time()) {
102
+
// Figure out which repositories we need to queue for an update.
103
+
foreach ($pullable as $id => $repository) {
104
+
$monogram = $repository->getMonogram();
105
+
106
+
if (isset($futures[$id])) {
107
+
$this->log(pht('Repository "%s" is currently updating.', $monogram));
110
108
continue;
111
109
}
112
110
113
-
$tracked = $repository->isTracked();
114
-
if (!$tracked) {
111
+
if (isset($queue[$id])) {
112
+
$this->log(pht('Repository "%s" is already queued.', $monogram));
115
113
continue;
116
114
}
117
115
118
-
$callsign = $repository->getCallsign();
116
+
$after = idx($retry_after, $id, 0);
117
+
if ($after > time()) {
118
+
$this->log(
119
+
pht(
120
+
'Repository "%s" is not due for an update for %s second(s).',
121
+
$monogram,
122
+
new PhutilNumber($after - time())));
123
+
continue;
124
+
}
119
125
120
-
try {
121
-
$this->log("Updating repository '{$callsign}'.");
126
+
if (!$after) {
127
+
$this->log(
128
+
pht(
129
+
'Scheduling repository "%s" for an initial update.',
130
+
$monogram));
131
+
} else {
132
+
$this->log(
133
+
pht(
134
+
'Scheduling repository "%s" for an update (%s seconds overdue).',
135
+
$monogram,
136
+
new PhutilNumber(time() - $after)));
137
+
}
122
138
123
-
$bin_dir = dirname(phutil_get_library_root('phabricator')).'/bin';
124
-
$flags = array();
125
-
if ($no_discovery) {
126
-
$flags[] = '--no-discovery';
127
-
}
139
+
$queue[$id] = $after;
140
+
}
128
141
129
-
list($stdout, $stderr) = execx(
130
-
'%s/repository update %Ls -- %s',
131
-
$bin_dir,
132
-
$flags,
133
-
$callsign);
142
+
// Process repositories in the order they became candidates for updates.
143
+
asort($queue);
134
144
135
-
if (strlen($stderr)) {
136
-
$stderr_msg = pht(
137
-
'Unexpected output while updating the %s repository: %s',
138
-
$callsign,
139
-
$stderr);
140
-
phlog($stderr_msg);
145
+
// Dequeue repositories until we hit maximum parallelism.
146
+
while ($queue && (count($futures) < $max_futures)) {
147
+
foreach ($queue as $id => $time) {
148
+
$repository = idx($pullable, $id);
149
+
if (!$repository) {
150
+
$this->log(
151
+
pht('Repository %s is no longer pullable; skipping.', $id));
152
+
break;
141
153
}
142
154
143
-
$sleep_for = $repository->getDetail('pull-frequency', $min_sleep);
144
-
$retry_after[$id] = time() + $sleep_for;
145
-
} catch (Exception $ex) {
146
-
$retry_after[$id] = time() + $min_sleep;
155
+
$monogram = $repository->getMonogram();
156
+
$this->log(pht('Starting update for repository "%s".', $monogram));
147
157
148
-
$proxy = new PhutilProxyException(
149
-
"Error while fetching changes to the '{$callsign}' repository.",
150
-
$ex);
151
-
phlog($proxy);
152
-
}
158
+
unset($queue[$id]);
159
+
$futures[$id] = $this->buildUpdateFuture(
160
+
$repository,
161
+
$no_discovery);
153
162
154
-
$this->stillWorking();
163
+
break;
164
+
}
155
165
}
156
166
157
-
if ($retry_after) {
158
-
$sleep_until = max(min($retry_after), time() + $min_sleep);
159
-
} else {
160
-
$sleep_until = time() + $min_sleep;
167
+
if ($queue) {
168
+
$this->log(
169
+
pht(
170
+
'Not enough process slots to schedule the other %s '.
171
+
'repository(s) for updates yet.',
172
+
new PhutilNumber(count($queue))));
161
173
}
162
174
163
-
while (($sleep_until - time()) > 0) {
164
-
$this->sleep(1);
165
-
if ($this->loadRepositoryUpdateMessages()) {
175
+
if ($futures) {
176
+
$iterator = id(new FutureIterator($futures))
177
+
->setUpdateInterval($min_sleep);
178
+
179
+
foreach ($iterator as $id => $future) {
180
+
$this->stillWorking();
181
+
182
+
if ($future === null) {
183
+
$this->log(pht('Waiting for updates to complete...'));
184
+
$this->stillWorking();
185
+
186
+
if ($this->loadRepositoryUpdateMessages()) {
187
+
$this->log(pht('Interrupted by pending updates!'));
188
+
break;
189
+
}
190
+
191
+
continue;
192
+
}
193
+
194
+
unset($futures[$id]);
195
+
$retry_after[$id] = $this->resolveUpdateFuture(
196
+
$pullable[$id],
197
+
$future,
198
+
$min_sleep);
199
+
200
+
// We have a free slot now, so go try to fill it.
166
201
break;
167
202
}
203
+
204
+
// Jump back into prioritization if we had any futures to deal with.
205
+
continue;
168
206
}
207
+
208
+
$this->waitForUpdates($min_sleep, $retry_after);
169
209
}
210
+
170
211
}
171
212
213
+
214
+
/**
215
+
* @task pull
216
+
*/
217
+
private function buildUpdateFuture(
218
+
PhabricatorRepository $repository,
219
+
$no_discovery) {
220
+
221
+
$bin = dirname(phutil_get_library_root('phabricator')).'/bin/repository';
222
+
223
+
$flags = array();
224
+
if ($no_discovery) {
225
+
$flags[] = '--no-discovery';
226
+
}
227
+
228
+
$callsign = $repository->getCallsign();
229
+
230
+
$future = new ExecFuture('%s update %Ls -- %s', $bin, $flags, $callsign);
231
+
232
+
// Sometimes, the underlying VCS commands will hang indefinitely. We've
233
+
// observed this occasionally with GitHub, and other users have observed
234
+
// it with other VCS servers.
235
+
236
+
// To limit the damage this can cause, kill the update out after a
237
+
// reasonable amount of time, under the assumption that it has hung.
238
+
239
+
// Since it's hard to know what a "reasonable" amount of time is given that
240
+
// users may be downloading a repository full of pirated movies over a
241
+
// potato, these limits are fairly generous. Repositories exceeding these
242
+
// limits can be manually pulled with `bin/repository update X`, which can
243
+
// just run for as long as it wants.
244
+
245
+
if ($repository->isImporting()) {
246
+
$timeout = phutil_units('4 hours in seconds');
247
+
} else {
248
+
$timeout = phutil_units('15 minutes in seconds');
249
+
}
250
+
251
+
$future->setTimeout($timeout);
252
+
253
+
return $future;
254
+
}
255
+
256
+
257
+
/**
258
+
* @task pull
259
+
*/
172
260
private function loadRepositoryUpdateMessages() {
173
261
$type_need_update = PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE;
174
262
return id(new PhabricatorRepositoryStatusMessage())
175
263
->loadAllWhere('statusType = %s', $type_need_update);
176
264
}
177
265
266
+
178
267
/**
179
268
* @task pull
180
269
*/
181
-
protected function loadRepositories(array $names) {
270
+
private function loadPullableRepositories(array $include, array $exclude) {
182
271
$query = id(new PhabricatorRepositoryQuery())
183
272
->setViewer($this->getViewer());
184
273
185
-
if ($names) {
186
-
$query->withCallsigns($names);
274
+
if ($include) {
275
+
$query->withCallsigns($include);
187
276
}
188
277
189
-
$repos = $query->execute();
278
+
$repositories = $query->execute();
190
279
191
-
if ($names) {
192
-
$by_callsign = mpull($repos, null, 'getCallsign');
193
-
foreach ($names as $name) {
280
+
if ($include) {
281
+
$by_callsign = mpull($repositories, null, 'getCallsign');
282
+
foreach ($include as $name) {
194
283
if (empty($by_callsign[$name])) {
195
284
throw new Exception(
196
285
"No repository exists with callsign '{$name}'!");
···
198
287
}
199
288
}
200
289
201
-
return $repos;
290
+
if ($exclude) {
291
+
$exclude = array_fuse($exclude);
292
+
foreach ($repositories as $key => $repository) {
293
+
if (isset($exclude[$repository->getCallsign()])) {
294
+
unset($repositories[$key]);
295
+
}
296
+
}
297
+
}
298
+
299
+
foreach ($repositories as $key => $repository) {
300
+
if (!$repository->isTracked()) {
301
+
unset($repositories[$key]);
302
+
}
303
+
}
304
+
305
+
// Shuffle the repositories, then re-key the array since shuffle()
306
+
// discards keys. This is mostly for startup, we'll use soft priorities
307
+
// later.
308
+
shuffle($repositories);
309
+
$repositories = mpull($repositories, null, 'getID');
310
+
311
+
return $repositories;
312
+
}
313
+
314
+
315
+
/**
316
+
* @task pull
317
+
*/
318
+
private function resolveUpdateFuture(
319
+
PhabricatorRepository $repository,
320
+
ExecFuture $future,
321
+
$min_sleep) {
322
+
323
+
$monogram = $repository->getMonogram();
324
+
325
+
$this->log(pht('Resolving update for "%s".', $monogram));
326
+
327
+
try {
328
+
list($stdout, $stderr) = $future->resolvex();
329
+
} catch (Exception $ex) {
330
+
$proxy = new PhutilProxyException(
331
+
pht(
332
+
'Error while updating the "%s" repository.',
333
+
$repository->getMonogram()),
334
+
$ex);
335
+
phlog($proxy);
336
+
337
+
return time() + $min_sleep;
338
+
}
339
+
340
+
if (strlen($stderr)) {
341
+
$stderr_msg = pht(
342
+
'Unexpected output while updating repository "%s": %s',
343
+
$monogram,
344
+
$stderr);
345
+
phlog($stderr_msg);
346
+
}
347
+
348
+
$sleep_for = (int)$repository->getDetail('pull-frequency', $min_sleep);
349
+
if ($sleep_for < $min_sleep) {
350
+
$sleep_for = $min_sleep;
351
+
}
352
+
353
+
return time() + $sleep_for;
354
+
}
355
+
356
+
357
+
358
+
/**
359
+
* Sleep for a short period of time, waiting for update messages from the
360
+
*
361
+
*
362
+
* @task pull
363
+
*/
364
+
private function waitForUpdates($min_sleep, array $retry_after) {
365
+
$this->log(
366
+
pht('No repositories need updates right now, sleeping...'));
367
+
368
+
$sleep_until = time() + $min_sleep;
369
+
if ($retry_after) {
370
+
$sleep_until = min($sleep_until, min($retry_after));
371
+
}
372
+
373
+
while (($sleep_until - time()) > 0) {
374
+
$sleep_duration = ($sleep_until - time());
375
+
376
+
$this->log(
377
+
pht(
378
+
'Sleeping for %s more second(s)...',
379
+
new PhutilNumber($sleep_duration)));
380
+
381
+
$this->sleep(1);
382
+
if ($this->loadRepositoryUpdateMessages()) {
383
+
$this->log(pht('Awakened from sleep by pending updates!'));
384
+
break;
385
+
}
386
+
}
202
387
}
203
388
204
389
}