@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Retain repository update cooldowns across daemon restarts
Summary:
Ref T11665. Fixes T7865. When we restart the daemons, the repository pull daemon currently resets the cooldowns on all of its pulls. This can generate a burst of initial load when restarting a lot of instance daemons (as in the Phacility cluster), described in T7865. This smooths things out so that recent pulls are considered, and any repositories which were waiting keep waiting.
Somewhat counterintuitively, hosted repositories write `TYPE_FETCH` status messages, so this should work equally well for hosted and observed repositories.
This also paves the way for better backoff behavior on repository errors, described in T11665. The error backoff now uses the same logic that the standard backoff does. The next change will make backoff computation consider recent errors.
(This is technically too large for repositories which have encountered one error and have a low commit rate, but I'll fix that in the following change; this is just a checkpoint on the way there.)
Test Plan: Ran `bin/phd debug pull`, saw the daemon compute reasonable windows based on previous pull activity.
Reviewers: chad
Maniphest Tasks: T7865, T11665
Differential Revision: https://secure.phabricator.com/D16574
+73
-28
+72
-27
src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
+72
-27
src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
···
102
102
$retry_after,
103
103
array_keys($pullable));
104
104
105
-
106
105
// Figure out which repositories we need to queue for an update.
107
106
foreach ($pullable as $id => $repository) {
108
-
$monogram = $repository->getMonogram();
107
+
$now = PhabricatorTime::getNow();
108
+
$display_name = $repository->getDisplayName();
109
109
110
110
if (isset($futures[$id])) {
111
-
$this->log(pht('Repository "%s" is currently updating.', $monogram));
111
+
$this->log(
112
+
pht(
113
+
'Repository "%s" is currently updating.',
114
+
$display_name));
112
115
continue;
113
116
}
114
117
115
118
if (isset($queue[$id])) {
116
-
$this->log(pht('Repository "%s" is already queued.', $monogram));
117
-
continue;
118
-
}
119
-
120
-
$after = idx($retry_after, $id, 0);
121
-
if ($after > time()) {
122
119
$this->log(
123
120
pht(
124
-
'Repository "%s" is not due for an update for %s second(s).',
125
-
$monogram,
126
-
new PhutilNumber($after - time())));
121
+
'Repository "%s" is already queued.',
122
+
$display_name));
127
123
continue;
128
124
}
129
125
126
+
$after = idx($retry_after, $id);
130
127
if (!$after) {
128
+
$smart_wait = $repository->loadUpdateInterval($min_sleep);
129
+
$last_update = $this->loadLastUpdate($repository);
130
+
131
+
$after = $last_update + $smart_wait;
132
+
$retry_after[$id] = $after;
133
+
131
134
$this->log(
132
135
pht(
133
-
'Scheduling repository "%s" for an initial update.',
134
-
$monogram));
135
-
} else {
136
+
'Scheduling repository "%s" with an update window of %s '.
137
+
'second(s). Last update was %s second(s) ago.',
138
+
$display_name,
139
+
new PhutilNumber($smart_wait),
140
+
new PhutilNumber($now - $last_update)));
141
+
}
142
+
143
+
if ($after > time()) {
136
144
$this->log(
137
145
pht(
138
-
'Scheduling repository "%s" for an update (%s seconds overdue).',
139
-
$monogram,
140
-
new PhutilNumber(time() - $after)));
146
+
'Repository "%s" is not due for an update for %s second(s).',
147
+
$display_name,
148
+
new PhutilNumber($after - $now)));
149
+
continue;
141
150
}
142
151
152
+
$this->log(
153
+
pht(
154
+
'Scheduling repository "%s" for an update (%s seconds overdue).',
155
+
$display_name,
156
+
new PhutilNumber($now - $after)));
157
+
143
158
$queue[$id] = $after;
144
159
}
145
160
···
157
172
continue;
158
173
}
159
174
160
-
$monogram = $repository->getMonogram();
161
-
$this->log(pht('Starting update for repository "%s".', $monogram));
175
+
$display_name = $repository->getDisplayName();
176
+
$this->log(
177
+
pht(
178
+
'Starting update for repository "%s".',
179
+
$display_name));
162
180
163
181
unset($queue[$id]);
164
182
$futures[$id] = $this->buildUpdateFuture(
···
299
317
/**
300
318
* @task pull
301
319
*/
320
+
private function loadLastUpdate(PhabricatorRepository $repository) {
321
+
$table = new PhabricatorRepositoryStatusMessage();
322
+
$conn = $table->establishConnection('r');
323
+
324
+
$epoch = queryfx_one(
325
+
$conn,
326
+
'SELECT MAX(epoch) last_update FROM %T
327
+
WHERE repositoryID = %d
328
+
AND statusType IN (%Ls)',
329
+
$table->getTableName(),
330
+
$repository->getID(),
331
+
array(
332
+
PhabricatorRepositoryStatusMessage::TYPE_INIT,
333
+
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
334
+
));
335
+
336
+
if ($epoch) {
337
+
return (int)$epoch['last_update'];
338
+
}
339
+
340
+
return PhabricatorTime::getNow();
341
+
}
342
+
343
+
/**
344
+
* @task pull
345
+
*/
302
346
private function loadPullableRepositories(
303
347
array $include,
304
348
array $exclude,
···
385
429
ExecFuture $future,
386
430
$min_sleep) {
387
431
388
-
$monogram = $repository->getMonogram();
432
+
$display_name = $repository->getDisplayName();
389
433
390
-
$this->log(pht('Resolving update for "%s".', $monogram));
434
+
$this->log(pht('Resolving update for "%s".', $display_name));
391
435
392
436
try {
393
437
list($stdout, $stderr) = $future->resolvex();
···
395
439
$proxy = new PhutilProxyException(
396
440
pht(
397
441
'Error while updating the "%s" repository.',
398
-
$repository->getMonogram()),
442
+
$display_name),
399
443
$ex);
400
444
phlog($proxy);
401
445
402
-
return time() + $min_sleep;
446
+
$smart_wait = $repository->loadUpdateInterval($min_sleep);
447
+
return PhabricatorTime::getNow() + $smart_wait;
403
448
}
404
449
405
450
if (strlen($stderr)) {
406
451
$stderr_msg = pht(
407
452
'Unexpected output while updating repository "%s": %s',
408
-
$monogram,
453
+
$display_name,
409
454
$stderr);
410
455
phlog($stderr_msg);
411
456
}
···
416
461
pht(
417
462
'Based on activity in repository "%s", considering a wait of %s '.
418
463
'seconds before update.',
419
-
$repository->getMonogram(),
464
+
$display_name,
420
465
new PhutilNumber($smart_wait)));
421
466
422
-
return time() + $smart_wait;
467
+
return PhabricatorTime::getNow() + $smart_wait;
423
468
}
424
469
425
470