@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add semi-generic rate limiting infrastructure
Summary:
This adds a system which basically keeps a record of recent actions, who took them, and how many "points" they were worth, like:
epriestley email.add 1 1233989813
epriestley email.add 1 1234298239
epriestley email.add 1 1238293981
We can use this to rate-limit actions by examining how many actions the user has taken in the past hour (i.e., their total score) and comparing that to an allowed limit.
One major thing I want to use this for is to limit the amount of error email we'll send to an email address. A big concern I have with sending more error email is that we'll end up in loops. We have some protections against this in headers already, but hard-limiting the system so it won't send more than a few errors to a particular address per hour should provide a reasonable secondary layer of protection.
This use case (where the "actor" needs to be an email address) is why the table uses strings + hashes instead of PHIDs. For external users, it might be appropriate to rate limit by cookies or IPs, too.
To prove it works, I rate limited adding email addresses. This is a very, very low-risk security thing where a user with an account can enumerate addresses (by checking if they get an error) and sort of spam/annoy people (by adding their address over and over again). Limiting them to 6 actions / hour should satisfy all real users while preventing these behaviors.
Test Plan:
This dialog is uggos but I'll fix that in a sec:
{F137406}
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: epriestley
Differential Revision: https://secure.phabricator.com/D8683
+297
+12
resources/sql/autopatches/20140402.actionlog.sql
+12
resources/sql/autopatches/20140402.actionlog.sql
···
1
+
CREATE TABLE {$NAMESPACE}_system.system_actionlog (
2
+
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
3
+
actorHash CHAR(12) NOT NULL COLLATE latin1_bin,
4
+
actorIdentity VARCHAR(255) NOT NULL COLLATE utf8_bin,
5
+
action CHAR(32) NOT NULL COLLATE utf8_bin,
6
+
score DOUBLE NOT NULL,
7
+
epoch INT UNSIGNED NOT NULL,
8
+
9
+
KEY `key_epoch` (epoch),
10
+
KEY `key_action` (actorHash, action, epoch)
11
+
12
+
) ENGINE=InnoDB, COLLATE utf8_general_ci;
+13
src/__phutil_library_map__.php
+13
src/__phutil_library_map__.php
···
2043
2043
'PhabricatorSearchWorker' => 'applications/search/worker/PhabricatorSearchWorker.php',
2044
2044
'PhabricatorSecurityConfigOptions' => 'applications/config/option/PhabricatorSecurityConfigOptions.php',
2045
2045
'PhabricatorSendGridConfigOptions' => 'applications/config/option/PhabricatorSendGridConfigOptions.php',
2046
+
'PhabricatorSettingsAddEmailAction' => 'applications/settings/action/PhabricatorSettingsAddEmailAction.php',
2046
2047
'PhabricatorSettingsAdjustController' => 'applications/settings/controller/PhabricatorSettingsAdjustController.php',
2047
2048
'PhabricatorSettingsMainController' => 'applications/settings/controller/PhabricatorSettingsMainController.php',
2048
2049
'PhabricatorSettingsPanel' => 'applications/settings/panel/PhabricatorSettingsPanel.php',
···
2142
2143
'PhabricatorSymbolNameLinter' => 'infrastructure/lint/hook/PhabricatorSymbolNameLinter.php',
2143
2144
'PhabricatorSyntaxHighlighter' => 'infrastructure/markup/PhabricatorSyntaxHighlighter.php',
2144
2145
'PhabricatorSyntaxHighlightingConfigOptions' => 'applications/config/option/PhabricatorSyntaxHighlightingConfigOptions.php',
2146
+
'PhabricatorSystemAction' => 'applications/system/action/PhabricatorSystemAction.php',
2147
+
'PhabricatorSystemActionEngine' => 'applications/system/engine/PhabricatorSystemActionEngine.php',
2148
+
'PhabricatorSystemActionGarbageCollector' => 'applications/system/garbagecollector/PhabricatorSystemActionGarbageCollector.php',
2149
+
'PhabricatorSystemActionLog' => 'applications/system/storage/PhabricatorSystemActionLog.php',
2150
+
'PhabricatorSystemActionRateLimitException' => 'applications/system/exception/PhabricatorSystemActionRateLimitException.php',
2151
+
'PhabricatorSystemDAO' => 'applications/system/storage/PhabricatorSystemDAO.php',
2145
2152
'PhabricatorTaskmasterDaemon' => 'infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php',
2146
2153
'PhabricatorTestCase' => 'infrastructure/testing/PhabricatorTestCase.php',
2147
2154
'PhabricatorTestController' => 'applications/base/controller/__tests__/PhabricatorTestController.php',
···
4906
4913
'PhabricatorSearchWorker' => 'PhabricatorWorker',
4907
4914
'PhabricatorSecurityConfigOptions' => 'PhabricatorApplicationConfigOptions',
4908
4915
'PhabricatorSendGridConfigOptions' => 'PhabricatorApplicationConfigOptions',
4916
+
'PhabricatorSettingsAddEmailAction' => 'PhabricatorSystemAction',
4909
4917
'PhabricatorSettingsAdjustController' => 'PhabricatorController',
4910
4918
'PhabricatorSettingsMainController' => 'PhabricatorController',
4911
4919
'PhabricatorSettingsPanelAccount' => 'PhabricatorSettingsPanel',
···
5006
5014
'PhabricatorSubscriptionsUIEventListener' => 'PhabricatorEventListener',
5007
5015
'PhabricatorSymbolNameLinter' => 'ArcanistXHPASTLintNamingHook',
5008
5016
'PhabricatorSyntaxHighlightingConfigOptions' => 'PhabricatorApplicationConfigOptions',
5017
+
'PhabricatorSystemActionEngine' => 'Phobject',
5018
+
'PhabricatorSystemActionGarbageCollector' => 'PhabricatorGarbageCollector',
5019
+
'PhabricatorSystemActionLog' => 'PhabricatorSystemDAO',
5020
+
'PhabricatorSystemActionRateLimitException' => 'Exception',
5021
+
'PhabricatorSystemDAO' => 'PhabricatorLiskDAO',
5009
5022
'PhabricatorTaskmasterDaemon' => 'PhabricatorDaemon',
5010
5023
'PhabricatorTestCase' => 'ArcanistPhutilTestCase',
5011
5024
'PhabricatorTestController' => 'PhabricatorController',
+17
src/aphront/configuration/AphrontDefaultApplicationConfiguration.php
+17
src/aphront/configuration/AphrontDefaultApplicationConfiguration.php
···
111
111
$user = new PhabricatorUser();
112
112
}
113
113
114
+
if ($ex instanceof PhabricatorSystemActionRateLimitException) {
115
+
$error_view = id(new AphrontErrorView())
116
+
->setErrors(array(pht('You are being rate limited.')));
117
+
118
+
$dialog = id(new AphrontDialogView())
119
+
->setTitle(pht('Slow Down!'))
120
+
->setUser($user)
121
+
->appendChild($error_view)
122
+
->appendParagraph($ex->getMessage())
123
+
->appendParagraph($ex->getRateExplanation())
124
+
->addCancelButton('/', pht('Okaaaaaaaaaaaaaay...'));
125
+
126
+
$response = new AphrontDialogResponse();
127
+
$response->setDialog($dialog);
128
+
return $response;
129
+
}
130
+
114
131
if ($ex instanceof PhabricatorPolicyException) {
115
132
116
133
if (!$user->isLoggedIn()) {
+20
src/applications/settings/action/PhabricatorSettingsAddEmailAction.php
+20
src/applications/settings/action/PhabricatorSettingsAddEmailAction.php
···
1
+
<?php
2
+
3
+
final class PhabricatorSettingsAddEmailAction extends PhabricatorSystemAction {
4
+
5
+
const TYPECONST = 'email.add';
6
+
7
+
public function getActionConstant() {
8
+
return self::TYPECONST;
9
+
}
10
+
11
+
public function getScoreThreshold() {
12
+
return 6 / phutil_units('1 hour in seconds');
13
+
}
14
+
15
+
public function getLimitExplanation() {
16
+
return pht(
17
+
'You are adding too many email addresses to your account too quickly.');
18
+
}
19
+
20
+
}
+5
src/applications/settings/panel/PhabricatorSettingsPanelEmailAddresses.php
+5
src/applications/settings/panel/PhabricatorSettingsPanelEmailAddresses.php
···
171
171
return id(new AphrontReloadResponse())->setURI($uri);
172
172
}
173
173
174
+
PhabricatorSystemActionEngine::willTakeAction(
175
+
array($user->getPHID()),
176
+
new PhabricatorSettingsAddEmailAction(),
177
+
1);
178
+
174
179
if (!strlen($email)) {
175
180
$e_email = pht('Required');
176
181
$errors[] = pht('Email is required.');
+40
src/applications/system/action/PhabricatorSystemAction.php
+40
src/applications/system/action/PhabricatorSystemAction.php
···
1
+
<?php
2
+
3
+
abstract class PhabricatorSystemAction {
4
+
5
+
abstract public function getActionConstant();
6
+
abstract public function getScoreThreshold();
7
+
8
+
public function shouldBlockActor($actor, $score) {
9
+
return ($score > $this->getScoreThreshold());
10
+
}
11
+
12
+
public function getLimitExplanation() {
13
+
return pht('You are performing too many actions too quickly.');
14
+
}
15
+
16
+
public function getRateExplanation($score) {
17
+
return pht(
18
+
'The maximum allowed rate for this action is %s. You are taking '.
19
+
'actions at a rate of %s.',
20
+
$this->formatRate($this->getScoreThreshold()),
21
+
$this->formatRate($score));
22
+
}
23
+
24
+
protected function formatRate($rate) {
25
+
if ($rate > 10) {
26
+
$str = pht('%d / second', $rate);
27
+
} else {
28
+
$rate *= 60;
29
+
if ($rate > 10) {
30
+
$str = pht('%d / minute', $rate);
31
+
} else {
32
+
$rate *= 60;
33
+
$str = pht('%d / hour', $rate);
34
+
}
35
+
}
36
+
37
+
return phutil_tag('strong', array(), $str);
38
+
}
39
+
40
+
}
+119
src/applications/system/engine/PhabricatorSystemActionEngine.php
+119
src/applications/system/engine/PhabricatorSystemActionEngine.php
···
1
+
<?php
2
+
3
+
final class PhabricatorSystemActionEngine extends Phobject {
4
+
5
+
public static function willTakeAction(
6
+
array $actors,
7
+
PhabricatorSystemAction $action,
8
+
$score) {
9
+
10
+
// If the score for this action is negative, we're giving the user a credit,
11
+
// so don't bother checking if they're blocked or not.
12
+
if ($score >= 0) {
13
+
$blocked = self::loadBlockedActors($actors, $action, $score);
14
+
if ($blocked) {
15
+
foreach ($blocked as $actor => $actor_score) {
16
+
throw new PhabricatorSystemActionRateLimitException(
17
+
$action,
18
+
$actor_score + ($score / self::getWindow()));
19
+
}
20
+
}
21
+
}
22
+
23
+
self::recordAction($actors, $action, $score);
24
+
}
25
+
26
+
public static function loadBlockedActors(
27
+
array $actors,
28
+
PhabricatorSystemAction $action) {
29
+
30
+
$scores = self::loadScores($actors, $action);
31
+
32
+
$blocked = array();
33
+
foreach ($scores as $actor => $score) {
34
+
if ($action->shouldBlockActor($actor, $score)) {
35
+
$blocked[$actor] = $score;
36
+
}
37
+
}
38
+
39
+
return $blocked;
40
+
}
41
+
42
+
public static function loadScores(
43
+
array $actors,
44
+
PhabricatorSystemAction $action) {
45
+
46
+
if (!$actors) {
47
+
return array();
48
+
}
49
+
50
+
$actor_hashes = array();
51
+
foreach ($actors as $actor) {
52
+
$actor_hashes[] = PhabricatorHash::digestForIndex($actor);
53
+
}
54
+
55
+
$log = new PhabricatorSystemActionLog();
56
+
57
+
$window = self::getWindow();
58
+
59
+
$conn_r = $log->establishConnection('r');
60
+
$scores = queryfx_all(
61
+
$conn_r,
62
+
'SELECT actorIdentity, SUM(score) totalScore FROM %T
63
+
WHERE action = %s AND actorHash IN (%Ls)
64
+
AND epoch >= %d GROUP BY actorHash',
65
+
$log->getTableName(),
66
+
$action->getActionConstant(),
67
+
$actor_hashes,
68
+
(time() - $window));
69
+
70
+
$scores = ipull($scores, 'totalScore', 'actorIdentity');
71
+
72
+
foreach ($scores as $key => $score) {
73
+
$scores[$key] = $score / $window;
74
+
}
75
+
76
+
$scores = $scores + array_fill_keys($actors, 0);
77
+
78
+
return $scores;
79
+
}
80
+
81
+
private static function recordAction(
82
+
array $actors,
83
+
PhabricatorSystemAction $action,
84
+
$score) {
85
+
86
+
$log = new PhabricatorSystemActionLog();
87
+
$conn_w = $log->establishConnection('w');
88
+
89
+
$sql = array();
90
+
foreach ($actors as $actor) {
91
+
$sql[] = qsprintf(
92
+
$conn_w,
93
+
'(%s, %s, %s, %f, %d)',
94
+
PhabricatorHash::digestForIndex($actor),
95
+
$actor,
96
+
$action->getActionConstant(),
97
+
$score,
98
+
time());
99
+
}
100
+
101
+
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
102
+
queryfx(
103
+
$conn_w,
104
+
'INSERT INTO %T (actorHash, actorIdentity, action, score, epoch)
105
+
VALUES %Q',
106
+
$log->getTableName(),
107
+
$chunk);
108
+
}
109
+
}
110
+
111
+
private static function getWindow() {
112
+
// Limit queries to the last hour of data so we don't need to look at as
113
+
// many rows. We can use an arbitrarily larger window instead (we normalize
114
+
// scores to actions per second) but all the actions we care about limiting
115
+
// have a limit much higher than one action per hour.
116
+
return phutil_units('1 hour in seconds');
117
+
}
118
+
119
+
}
+18
src/applications/system/exception/PhabricatorSystemActionRateLimitException.php
+18
src/applications/system/exception/PhabricatorSystemActionRateLimitException.php
···
1
+
<?php
2
+
3
+
final class PhabricatorSystemActionRateLimitException extends Exception {
4
+
5
+
private $action;
6
+
private $score;
7
+
8
+
public function __construct(PhabricatorSystemAction $action, $score) {
9
+
$this->action = $action;
10
+
$this->score = $score;
11
+
parent::__construct($action->getLimitExplanation());
12
+
}
13
+
14
+
public function getRateExplanation() {
15
+
return $this->action->getRateExplanation($this->score);
16
+
}
17
+
18
+
}
+21
src/applications/system/garbagecollector/PhabricatorSystemActionGarbageCollector.php
+21
src/applications/system/garbagecollector/PhabricatorSystemActionGarbageCollector.php
···
1
+
<?php
2
+
3
+
final class PhabricatorSystemActionGarbageCollector
4
+
extends PhabricatorGarbageCollector {
5
+
6
+
public function collectGarbage() {
7
+
$ttl = phutil_units('3 days in seconds');
8
+
9
+
$table = new PhabricatorSystemActionLog();
10
+
$conn_w = $table->establishConnection('w');
11
+
12
+
queryfx(
13
+
$conn_w,
14
+
'DELETE FROM %T WHERE epoch < %d LIMIT 100',
15
+
$table->getTableName(),
16
+
time() - $ttl);
17
+
18
+
return ($conn_w->getAffectedRows() == 100);
19
+
}
20
+
21
+
}
+22
src/applications/system/storage/PhabricatorSystemActionLog.php
+22
src/applications/system/storage/PhabricatorSystemActionLog.php
···
1
+
<?php
2
+
3
+
final class PhabricatorSystemActionLog extends PhabricatorSystemDAO {
4
+
5
+
protected $actorHash;
6
+
protected $actorIdentity;
7
+
protected $action;
8
+
protected $score;
9
+
protected $epoch;
10
+
11
+
public function getConfiguration() {
12
+
return array(
13
+
self::CONFIG_TIMESTAMPS => false,
14
+
) + parent::getConfiguration();
15
+
}
16
+
17
+
public function setActorIdentity($identity) {
18
+
$this->setActorHash(PhabricatorHash::digestForIndex($identity));
19
+
return parent::setActorIdentity($identity);
20
+
}
21
+
22
+
}
+9
src/applications/system/storage/PhabricatorSystemDAO.php
+9
src/applications/system/storage/PhabricatorSystemDAO.php