@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
When repositories hit pull errors, stop updating them as frequently
Summary:
Ref T11665. Currently, when a repository hits an error, we retry it after 15s. This is correct if the error was temporary/transient/config-related (e.g., bad network or administrator setting up credentials) but not so great if the error is long-lasting (completely bad authentication, invalid URI, etc), as it can pile up to a meaningful amount of unnecessary load over time.
Instead, record how many times in a row we've hit an error and adjust backoff behavior: first error is 15s, then 30s, 45s, etc.
Additionally, when computing the backoff for an empty repository, use the repository creation time as though it was the most recent commit. This is a good proxy which gives us reasonable backoff behavior.
This required removing the `CODE_WORKING` messages, since they would have reset the error count. We could restore them (as a different type of message), but I think they aren't particularly useful since cloning usually doesn't take too long and there's more status information avilable now than there was when this stuff was written.
Test Plan:
- Ran `bin/phd debug pull`.
- Saw sensible, increasing backoffs selected for repositories with errors.
- Saw sensible backoffs selected for empty repositories.
Reviewers: chad
Maniphest Tasks: T11665
Differential Revision: https://secure.phabricator.com/D16575
+71
-35
+2
resources/sql/autopatches/20160919.repo.messagecount.sql
+2
resources/sql/autopatches/20160919.repo.messagecount.sql
+1
-8
src/applications/diffusion/management/DiffusionRepositoryStatusManagementPanel.php
+1
-8
src/applications/diffusion/management/DiffusionRepositoryStatusManagementPanel.php
···
359
359
return $view;
360
360
}
361
361
break;
362
-
case PhabricatorRepositoryStatusMessage::CODE_WORKING:
362
+
default:
363
363
$view->addItem(
364
364
id(new PHUIStatusItemView())
365
365
->setIcon(PHUIStatusItemView::ICON_CLOCK, 'green')
366
366
->setTarget(pht('Initializing Working Copy'))
367
367
->setNote(pht('Daemons are initializing the working copy.')));
368
-
return $view;
369
-
default:
370
-
$view->addItem(
371
-
id(new PHUIStatusItemView())
372
-
->setIcon(PHUIStatusItemView::ICON_WARNING, 'red')
373
-
->setTarget(pht('Unknown Init Status'))
374
-
->setNote($message->getStatusCode()));
375
368
return $view;
376
369
}
377
370
} else {
-2
src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
-2
src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
+66
-24
src/applications/repository/storage/PhabricatorRepository.php
+66
-24
src/applications/repository/storage/PhabricatorRepository.php
···
1649
1649
$this->getID(),
1650
1650
$status_type);
1651
1651
} else {
1652
+
// If the existing message has the same code (e.g., we just hit an
1653
+
// error and also previously hit an error) we increment the message
1654
+
// count by 1. This allows us to determine how many times in a row
1655
+
// we've run into an error.
1656
+
1652
1657
queryfx(
1653
1658
$conn_w,
1654
1659
'INSERT INTO %T
1655
-
(repositoryID, statusType, statusCode, parameters, epoch)
1656
-
VALUES (%d, %s, %s, %s, %d)
1660
+
(repositoryID, statusType, statusCode, parameters, epoch,
1661
+
messageCount)
1662
+
VALUES (%d, %s, %s, %s, %d, %d)
1657
1663
ON DUPLICATE KEY UPDATE
1658
1664
statusCode = VALUES(statusCode),
1659
1665
parameters = VALUES(parameters),
1660
-
epoch = VALUES(epoch)',
1666
+
epoch = VALUES(epoch),
1667
+
messageCount =
1668
+
IF(
1669
+
statusCode = VALUES(statusCode),
1670
+
messageCount + 1,
1671
+
VALUES(messageCount))',
1661
1672
$table_name,
1662
1673
$this->getID(),
1663
1674
$status_type,
1664
1675
$status_code,
1665
1676
json_encode($parameters),
1666
-
time());
1677
+
time(),
1678
+
1);
1667
1679
}
1668
1680
1669
1681
return $this;
···
1738
1750
* @return int Repository update interval, in seconds.
1739
1751
*/
1740
1752
public function loadUpdateInterval($minimum = 15) {
1753
+
// First, check if we've hit errors recently. If we have, wait one period
1754
+
// for each consecutive error. Normally, this corresponds to a backoff of
1755
+
// 15s, 30s, 45s, etc.
1756
+
1757
+
$message_table = new PhabricatorRepositoryStatusMessage();
1758
+
$conn = $message_table->establishConnection('r');
1759
+
$error_count = queryfx_one(
1760
+
$conn,
1761
+
'SELECT MAX(messageCount) error_count FROM %T
1762
+
WHERE repositoryID = %d
1763
+
AND statusType IN (%Ls)
1764
+
AND statusCode IN (%Ls)',
1765
+
$message_table->getTableName(),
1766
+
$this->getID(),
1767
+
array(
1768
+
PhabricatorRepositoryStatusMessage::TYPE_INIT,
1769
+
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
1770
+
),
1771
+
array(
1772
+
PhabricatorRepositoryStatusMessage::CODE_ERROR,
1773
+
));
1774
+
1775
+
$error_count = (int)$error_count['error_count'];
1776
+
if ($error_count > 0) {
1777
+
return (int)($minimum * $error_count);
1778
+
}
1779
+
1741
1780
// If a repository is still importing, always pull it as frequently as
1742
1781
// possible. This prevents us from hanging for a long time at 99.9% when
1743
1782
// importing an inactive repository.
···
1758
1797
$window_start);
1759
1798
if ($last_commit) {
1760
1799
$time_since_commit = ($window_start - $last_commit['epoch']);
1800
+
} else {
1801
+
// If the repository has no commits, treat the creation date as
1802
+
// though it were the date of the last commit. This makes empty
1803
+
// repositories update quickly at first but slow down over time
1804
+
// if they don't see any activity.
1805
+
$time_since_commit = ($window_start - $this->getDateCreated());
1806
+
}
1761
1807
1762
-
$last_few_days = phutil_units('3 days in seconds');
1808
+
$last_few_days = phutil_units('3 days in seconds');
1763
1809
1764
-
if ($time_since_commit <= $last_few_days) {
1765
-
// For repositories with activity in the recent past, we wait one
1766
-
// extra second for every 10 minutes since the last commit. This
1767
-
// shorter backoff is intended to handle weekends and other short
1768
-
// breaks from development.
1769
-
$smart_wait = ($time_since_commit / 600);
1770
-
} else {
1771
-
// For repositories without recent activity, we wait one extra second
1772
-
// for every 4 minutes since the last commit. This longer backoff
1773
-
// handles rarely used repositories, up to the maximum.
1774
-
$smart_wait = ($time_since_commit / 240);
1775
-
}
1776
-
1777
-
// We'll never wait more than 6 hours to pull a repository.
1778
-
$longest_wait = phutil_units('6 hours in seconds');
1779
-
$smart_wait = min($smart_wait, $longest_wait);
1780
-
1781
-
$smart_wait = max($minimum, $smart_wait);
1810
+
if ($time_since_commit <= $last_few_days) {
1811
+
// For repositories with activity in the recent past, we wait one
1812
+
// extra second for every 10 minutes since the last commit. This
1813
+
// shorter backoff is intended to handle weekends and other short
1814
+
// breaks from development.
1815
+
$smart_wait = ($time_since_commit / 600);
1782
1816
} else {
1783
-
$smart_wait = $minimum;
1817
+
// For repositories without recent activity, we wait one extra second
1818
+
// for every 4 minutes since the last commit. This longer backoff
1819
+
// handles rarely used repositories, up to the maximum.
1820
+
$smart_wait = ($time_since_commit / 240);
1784
1821
}
1822
+
1823
+
// We'll never wait more than 6 hours to pull a repository.
1824
+
$longest_wait = phutil_units('6 hours in seconds');
1825
+
$smart_wait = min($smart_wait, $longest_wait);
1826
+
$smart_wait = max($minimum, $smart_wait);
1785
1827
1786
1828
return (int)$smart_wait;
1787
1829
}
+2
-1
src/applications/repository/storage/PhabricatorRepositoryStatusMessage.php
+2
-1
src/applications/repository/storage/PhabricatorRepositoryStatusMessage.php
···
8
8
const TYPE_NEEDS_UPDATE = 'needs-update';
9
9
10
10
const CODE_ERROR = 'error';
11
-
const CODE_WORKING = 'working';
12
11
const CODE_OKAY = 'okay';
13
12
14
13
protected $repositoryID;
···
16
15
protected $statusCode;
17
16
protected $parameters = array();
18
17
protected $epoch;
18
+
protected $messageCount;
19
19
20
20
protected function getConfiguration() {
21
21
return array(
···
26
26
self::CONFIG_COLUMN_SCHEMA => array(
27
27
'statusType' => 'text32',
28
28
'statusCode' => 'text32',
29
+
'messageCount' => 'uint32',
29
30
),
30
31
self::CONFIG_KEY_SCHEMA => array(
31
32
'repositoryID' => array(