@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Provide a general-purpose, modular user cache for settings and other similar data
Summary:
Ref T4103. Currently, we issue a `SELECT * FROM user_preferences ... WHERE userPHID = ...` on every page to load the viewer's settings.
There are several other questionable data accesses on every page too, most of which could benefit from improved caching strategies (see T4103#178122).
This query will soon get more expensive, since it may need to load several objects (e.g., the user's settings and their "role profile" settings). Although we could put that data on the User and do both in one query, it's nicer to put it on the Preferences object ("This inherits from profile X") which means we need to do several queries.
Rather than paying a greater price, we can cheat this stuff into the existing query where we load the user's session by providing a user cache table and doing some JOIN magic. This lets us issue one query and try to get cache hits on a bunch of caches cheaply (well, we'll be in trouble at the MySQL JOIN limit of 61 tables, but have some headroom).
For now, just get it working:
- Add the table.
- Try to get user settings "for free" when we load the session.
- If we miss, fill user settings into the cache on-demand.
- We only use this in one place (DarkConsole) for now. I'll use it more widely in the next diff.
Test Plan:
- Loaded page as logged-in user.
- Loaded page as logged-out user.
- Examined session query to see cache joins.
- Changed settings, saw database cache fill.
- Toggled DarkConsole on and off.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T4103
Differential Revision: https://secure.phabricator.com/D16001
+422
-10
+11
resources/sql/autopatches/20160601.user.01.cache.sql
+11
resources/sql/autopatches/20160601.user.01.cache.sql
···
1
+
CREATE TABLE {$NAMESPACE}_user.user_cache (
2
+
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
3
+
userPHID VARBINARY(64) NOT NULL,
4
+
cacheIndex BINARY(12) NOT NULL,
5
+
cacheKey VARCHAR(255) NOT NULL COLLATE {$COLLATE_TEXT},
6
+
cacheData LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT},
7
+
cacheType VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT},
8
+
UNIQUE KEY `key_usercache` (userPHID, cacheIndex),
9
+
KEY `key_cachekey` (cacheIndex),
10
+
KEY `key_cachetype` (cacheType)
11
+
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};
+7
-1
src/__phutil_library_map__.php
+7
-1
src/__phutil_library_map__.php
···
3596
3596
'PhabricatorUnsubscribedFromObjectEdgeType' => 'applications/transactions/edges/PhabricatorUnsubscribedFromObjectEdgeType.php',
3597
3597
'PhabricatorUser' => 'applications/people/storage/PhabricatorUser.php',
3598
3598
'PhabricatorUserBlurbField' => 'applications/people/customfield/PhabricatorUserBlurbField.php',
3599
+
'PhabricatorUserCache' => 'applications/people/storage/PhabricatorUserCache.php',
3600
+
'PhabricatorUserCacheType' => 'applications/people/cache/PhabricatorUserCacheType.php',
3599
3601
'PhabricatorUserCardView' => 'applications/people/view/PhabricatorUserCardView.php',
3600
3602
'PhabricatorUserConfigOptions' => 'applications/people/config/PhabricatorUserConfigOptions.php',
3601
3603
'PhabricatorUserConfiguredCustomField' => 'applications/people/customfield/PhabricatorUserConfiguredCustomField.php',
···
3614
3616
'PhabricatorUserLogView' => 'applications/people/view/PhabricatorUserLogView.php',
3615
3617
'PhabricatorUserPHIDResolver' => 'applications/phid/resolver/PhabricatorUserPHIDResolver.php',
3616
3618
'PhabricatorUserPreferences' => 'applications/settings/storage/PhabricatorUserPreferences.php',
3619
+
'PhabricatorUserPreferencesCacheType' => 'applications/people/cache/PhabricatorUserPreferencesCacheType.php',
3617
3620
'PhabricatorUserPreferencesEditor' => 'applications/settings/editor/PhabricatorUserPreferencesEditor.php',
3618
3621
'PhabricatorUserPreferencesPHIDType' => 'applications/settings/phid/PhabricatorUserPreferencesPHIDType.php',
3619
3622
'PhabricatorUserPreferencesQuery' => 'applications/settings/query/PhabricatorUserPreferencesQuery.php',
···
8368
8371
'PhabricatorConduitResultInterface',
8369
8372
),
8370
8373
'PhabricatorUserBlurbField' => 'PhabricatorUserCustomField',
8374
+
'PhabricatorUserCache' => 'PhabricatorUserDAO',
8375
+
'PhabricatorUserCacheType' => 'Phobject',
8371
8376
'PhabricatorUserCardView' => 'AphrontTagView',
8372
8377
'PhabricatorUserConfigOptions' => 'PhabricatorApplicationConfigOptions',
8373
8378
'PhabricatorUserConfiguredCustomField' => array(
···
8397
8402
'PhabricatorDestructibleInterface',
8398
8403
'PhabricatorApplicationTransactionInterface',
8399
8404
),
8400
-
'PhabricatorUserPreferencesEditor' => 'AlmanacEditor',
8405
+
'PhabricatorUserPreferencesCacheType' => 'PhabricatorUserCacheType',
8406
+
'PhabricatorUserPreferencesEditor' => 'PhabricatorApplicationTransactionEditor',
8401
8407
'PhabricatorUserPreferencesPHIDType' => 'PhabricatorPHIDType',
8402
8408
'PhabricatorUserPreferencesQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
8403
8409
'PhabricatorUserPreferencesTransaction' => 'PhabricatorApplicationTransaction',
+84
-5
src/applications/auth/engine/PhabricatorAuthSessionEngine.php
+84
-5
src/applications/auth/engine/PhabricatorAuthSessionEngine.php
···
7
7
* @task hisec High Security
8
8
* @task partial Partial Sessions
9
9
* @task onetime One Time Login URIs
10
+
* @task cache User Cache
10
11
*/
11
12
final class PhabricatorAuthSessionEngine extends Phobject {
12
13
···
111
112
$conn_r = $session_table->establishConnection('r');
112
113
$session_key = PhabricatorHash::digest($session_token);
113
114
114
-
// NOTE: We're being clever here because this happens on every page load,
115
-
// and by joining we can save a query. This might be getting too clever
116
-
// for its own good, though...
115
+
$cache_parts = $this->getUserCacheQueryParts($conn_r);
116
+
list($cache_selects, $cache_joins, $cache_map) = $cache_parts;
117
117
118
118
$info = queryfx_one(
119
119
$conn_r,
···
125
125
s.isPartial AS s_isPartial,
126
126
s.signedLegalpadDocuments as s_signedLegalpadDocuments,
127
127
u.*
128
+
%Q
128
129
FROM %T u JOIN %T s ON u.phid = s.userPHID
129
-
AND s.type = %s AND s.sessionKey = %s',
130
+
AND s.type = %s AND s.sessionKey = %s %Q',
131
+
$cache_selects,
130
132
$user_table->getTableName(),
131
133
$session_table->getTableName(),
132
134
$session_type,
133
-
$session_key);
135
+
$session_key,
136
+
$cache_joins);
134
137
135
138
if (!$info) {
136
139
return null;
···
141
144
'sessionKey' => $session_key,
142
145
'type' => $session_type,
143
146
);
147
+
148
+
$cache_raw = array_fill_keys($cache_map, null);
144
149
foreach ($info as $key => $value) {
145
150
if (strncmp($key, 's_', 2) === 0) {
146
151
unset($info[$key]);
147
152
$session_dict[substr($key, 2)] = $value;
153
+
continue;
154
+
}
155
+
156
+
if (isset($cache_map[$key])) {
157
+
unset($info[$key]);
158
+
$cache_raw[$cache_map[$key]] = $value;
159
+
continue;
148
160
}
149
161
}
150
162
151
163
$user = $user_table->loadFromArray($info);
164
+
165
+
$user->attachRawCacheData($cache_raw);
166
+
152
167
switch ($session_type) {
153
168
case PhabricatorAuthSession::TYPE_WEB:
154
169
// Explicitly prevent bots and mailing lists from establishing web
···
730
745
}
731
746
732
747
return PhabricatorHash::digest(implode(':', $parts));
748
+
}
749
+
750
+
751
+
/* -( User Cache )--------------------------------------------------------- */
752
+
753
+
754
+
/**
755
+
* @task cache
756
+
*/
757
+
private function getUserCacheQueryParts(AphrontDatabaseConnection $conn) {
758
+
$cache_selects = array();
759
+
$cache_joins = array();
760
+
$cache_map = array();
761
+
762
+
$keys = array();
763
+
764
+
$cache_types = PhabricatorUserCacheType::getAllCacheTypes();
765
+
foreach ($cache_types as $cache_type) {
766
+
foreach ($cache_type->getAutoloadKeys() as $autoload_key) {
767
+
$keys[] = $autoload_key;
768
+
}
769
+
}
770
+
771
+
$cache_table = id(new PhabricatorUserCache())->getTableName();
772
+
773
+
$cache_idx = 1;
774
+
foreach ($keys as $key) {
775
+
$join_as = 'ucache_'.$cache_idx;
776
+
$select_as = 'ucache_'.$cache_idx.'_v';
777
+
778
+
$cache_selects[] = qsprintf(
779
+
$conn,
780
+
'%T.cacheData %T',
781
+
$join_as,
782
+
$select_as);
783
+
784
+
$cache_joins[] = qsprintf(
785
+
$conn,
786
+
'LEFT JOIN %T AS %T ON u.phid = %T.userPHID
787
+
AND %T.cacheIndex = %s',
788
+
$cache_table,
789
+
$join_as,
790
+
$join_as,
791
+
$join_as,
792
+
PhabricatorHash::digestForIndex($key));
793
+
794
+
$cache_map[$select_as] = $key;
795
+
796
+
$cache_idx++;
797
+
}
798
+
799
+
if ($cache_selects) {
800
+
$cache_selects = ', '.implode(', ', $cache_selects);
801
+
} else {
802
+
$cache_selects = '';
803
+
}
804
+
805
+
if ($cache_joins) {
806
+
$cache_joins = implode(' ', $cache_joins);
807
+
} else {
808
+
$cache_joins = '';
809
+
}
810
+
811
+
return array($cache_selects, $cache_joins, $cache_map);
733
812
}
734
813
735
814
}
+2
-3
src/applications/base/controller/PhabricatorController.php
+2
-3
src/applications/base/controller/PhabricatorController.php
···
106
106
107
107
PhabricatorEnv::setLocaleCode($user->getTranslation());
108
108
109
-
$preferences = $user->loadPreferences();
110
109
if (PhabricatorEnv::getEnvConfig('darkconsole.enabled')) {
111
-
$dark_console = PhabricatorUserPreferences::PREFERENCE_DARK_CONSOLE;
112
-
if ($preferences->getPreference($dark_console) ||
110
+
$dark_console = PhabricatorDarkConsoleSetting::SETTINGKEY;
111
+
if ($user->getUserSetting($dark_console) ||
113
112
PhabricatorEnv::getEnvConfig('darkconsole.always-on')) {
114
113
$console = new DarkConsoleCore();
115
114
$request->getApplicationConfiguration()->setConsole($console);
+70
src/applications/people/cache/PhabricatorUserCacheType.php
+70
src/applications/people/cache/PhabricatorUserCacheType.php
···
1
+
<?php
2
+
3
+
abstract class PhabricatorUserCacheType extends Phobject {
4
+
5
+
final public function getViewer() {
6
+
return PhabricatorUser::getOmnipotentUser();
7
+
}
8
+
9
+
public function getAutoloadKeys() {
10
+
return array();
11
+
}
12
+
13
+
public function canManageKey($key) {
14
+
return false;
15
+
}
16
+
17
+
public function getDefaultValue() {
18
+
return array();
19
+
}
20
+
21
+
public function getValueFromStorage($value) {
22
+
return phutil_json_decode($value);
23
+
}
24
+
25
+
public function getValueForStorage($value) {
26
+
return phutil_json_encode($value);
27
+
}
28
+
29
+
public function newValueForUsers($key, array $users) {
30
+
return array();
31
+
}
32
+
33
+
final public function getUserCacheType() {
34
+
return $this->getPhobjectClassConstant('CACHETYPE');
35
+
}
36
+
37
+
public static function getAllCacheTypes() {
38
+
return id(new PhutilClassMapQuery())
39
+
->setAncestorClass(__CLASS__)
40
+
->setUniqueMethod('getUserCacheType')
41
+
->execute();
42
+
}
43
+
44
+
public static function getCacheTypeForKey($key) {
45
+
$all = self::getAllCacheTypes();
46
+
47
+
foreach ($all as $type) {
48
+
if ($type->canManageKey($key)) {
49
+
return $type;
50
+
}
51
+
}
52
+
53
+
return null;
54
+
}
55
+
56
+
public static function requireCacheTypeForKey($key) {
57
+
$type = self::getCacheTypeForKey($key);
58
+
59
+
if (!$type) {
60
+
throw new Exception(
61
+
pht(
62
+
'Failed to load UserCacheType to manage key "%s". This cache type '.
63
+
'is required.',
64
+
$key));
65
+
}
66
+
67
+
return $type;
68
+
}
69
+
70
+
}
+31
src/applications/people/cache/PhabricatorUserPreferencesCacheType.php
+31
src/applications/people/cache/PhabricatorUserPreferencesCacheType.php
···
1
+
<?php
2
+
3
+
final class PhabricatorUserPreferencesCacheType
4
+
extends PhabricatorUserCacheType {
5
+
6
+
const CACHETYPE = 'preferences';
7
+
8
+
const KEY_PREFERENCES = 'user.preferences.v1';
9
+
10
+
public function getAutoloadKeys() {
11
+
return array(
12
+
self::KEY_PREFERENCES,
13
+
);
14
+
}
15
+
16
+
public function canManageKey($key) {
17
+
return ($key === self::KEY_PREFERENCES);
18
+
}
19
+
20
+
public function newValueForUsers($key, array $users) {
21
+
$viewer = $this->getViewer();
22
+
23
+
$preferences = id(new PhabricatorUserPreferencesQuery())
24
+
->setViewer($viewer)
25
+
->withUserPHIDs(mpull($users, 'getPHID'))
26
+
->execute();
27
+
28
+
return mpull($preferences, 'getPreferences', 'getUserPHID');
29
+
}
30
+
31
+
}
+73
src/applications/people/storage/PhabricatorUser.php
+73
src/applications/people/storage/PhabricatorUser.php
···
5
5
* @task image-cache Profile Image Cache
6
6
* @task factors Multi-Factor Authentication
7
7
* @task handles Managing Handles
8
+
* @task cache User Cache
8
9
*/
9
10
final class PhabricatorUser
10
11
extends PhabricatorUserDAO
···
61
62
62
63
private $alternateCSRFString = self::ATTACHABLE;
63
64
private $session = self::ATTACHABLE;
65
+
private $rawCacheData = array();
66
+
private $usableCacheData = array();
64
67
65
68
private $authorities = array();
66
69
private $handlePool;
···
485
488
'userPHID',
486
489
'getPHID',
487
490
'(isPrimary = 1)');
491
+
}
492
+
493
+
public function getUserSetting($key) {
494
+
$settings_key = PhabricatorUserPreferencesCacheType::KEY_PREFERENCES;
495
+
$settings = $this->requireCacheData($settings_key);
496
+
497
+
if (array_key_exists($key, $settings)) {
498
+
return $settings[$key];
499
+
}
500
+
501
+
$defaults = PhabricatorSetting::getAllEnabledSettings($this);
502
+
if (isset($defaults[$key])) {
503
+
return $defaults[$key]->getSettingDefaultValue();
504
+
}
505
+
506
+
return null;
488
507
}
489
508
490
509
public function loadPreferences() {
···
1460
1479
return array();
1461
1480
}
1462
1481
1482
+
1483
+
/* -( User Cache )--------------------------------------------------------- */
1484
+
1485
+
1486
+
/**
1487
+
* @task cache
1488
+
*/
1489
+
public function attachRawCacheData(array $data) {
1490
+
$this->rawCacheData = $data + $this->rawCacheData;
1491
+
return $this;
1492
+
}
1493
+
1494
+
1495
+
/**
1496
+
* @task cache
1497
+
*/
1498
+
protected function requireCacheData($key) {
1499
+
if (isset($this->usableCacheData[$key])) {
1500
+
return $this->usableCacheData[$key];
1501
+
}
1502
+
1503
+
$type = PhabricatorUserCacheType::requireCacheTypeForKey($key);
1504
+
1505
+
if (isset($this->rawCacheData[$key])) {
1506
+
$raw_value = $this->rawCacheData[$key];
1507
+
1508
+
$usable_value = $type->getValueFromStorage($raw_value);
1509
+
$this->usableCacheData[$key] = $usable_value;
1510
+
1511
+
return $usable_value;
1512
+
}
1513
+
1514
+
$usable_value = $type->getDefaultValue();
1515
+
1516
+
$user_phid = $this->getPHID();
1517
+
if ($user_phid) {
1518
+
$map = $type->newValueForUsers($key, array($this));
1519
+
if (array_key_exists($user_phid, $map)) {
1520
+
$usable_value = $map[$user_phid];
1521
+
$raw_value = $type->getValueForStorage($usable_value);
1522
+
1523
+
$this->rawCacheData[$key] = $raw_value;
1524
+
PhabricatorUserCache::writeCache(
1525
+
$type,
1526
+
$key,
1527
+
$user_phid,
1528
+
$raw_value);
1529
+
}
1530
+
}
1531
+
1532
+
$this->usableCacheData[$key] = $usable_value;
1533
+
1534
+
return $usable_value;
1535
+
}
1463
1536
1464
1537
}
+110
src/applications/people/storage/PhabricatorUserCache.php
+110
src/applications/people/storage/PhabricatorUserCache.php
···
1
+
<?php
2
+
3
+
final class PhabricatorUserCache extends PhabricatorUserDAO {
4
+
5
+
protected $userPHID;
6
+
protected $cacheIndex;
7
+
protected $cacheKey;
8
+
protected $cacheData;
9
+
protected $cacheType;
10
+
11
+
protected function getConfiguration() {
12
+
return array(
13
+
self::CONFIG_TIMESTAMPS => false,
14
+
self::CONFIG_COLUMN_SCHEMA => array(
15
+
'cacheIndex' => 'bytes12',
16
+
'cacheKey' => 'text255',
17
+
'cacheData' => 'text',
18
+
'cacheType' => 'text32',
19
+
),
20
+
self::CONFIG_KEY_SCHEMA => array(
21
+
'key_usercache' => array(
22
+
'columns' => array('userPHID', 'cacheIndex'),
23
+
'unique' => true,
24
+
),
25
+
'key_cachekey' => array(
26
+
'columns' => array('cacheIndex'),
27
+
),
28
+
'key_cachetype' => array(
29
+
'columns' => array('cacheType'),
30
+
),
31
+
),
32
+
) + parent::getConfiguration();
33
+
}
34
+
35
+
public function save() {
36
+
$this->cacheIndex = Filesystem::digestForIndex($this->getCacheKey());
37
+
return parent::save();
38
+
}
39
+
40
+
public static function writeCache(
41
+
PhabricatorUserCacheType $type,
42
+
$key,
43
+
$user_phid,
44
+
$raw_value) {
45
+
46
+
if (PhabricatorEnv::isReadOnly()) {
47
+
return;
48
+
}
49
+
50
+
$table = new self();
51
+
$conn_w = $table->establishConnection('w');
52
+
53
+
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
54
+
55
+
queryfx(
56
+
$conn_w,
57
+
'INSERT INTO %T (userPHID, cacheIndex, cacheKey, cacheData, cacheType)
58
+
VALUES (%s, %s, %s, %s, %s)
59
+
ON DUPLICATE KEY UPDATE cacheData = VALUES(cacheData)',
60
+
$table->getTableName(),
61
+
$user_phid,
62
+
PhabricatorHash::digestForIndex($key),
63
+
$key,
64
+
$raw_value,
65
+
$type->getUserCacheType());
66
+
67
+
unset($unguarded);
68
+
}
69
+
70
+
public static function clearCache($key, $user_phid) {
71
+
if (PhabricatorEnv::isReadOnly()) {
72
+
return;
73
+
}
74
+
75
+
$table = new self();
76
+
$conn_w = $table->establishConnection('w');
77
+
78
+
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
79
+
80
+
queryfx(
81
+
$conn_w,
82
+
'DELETE FROM %T WHERE cacheIndex = %s AND userPHID = %s',
83
+
$table->getTableName(),
84
+
PhabricatorHash::digestForIndex($key),
85
+
$user_phid);
86
+
87
+
unset($unguarded);
88
+
}
89
+
90
+
91
+
public static function clearCacheForAllUsers($key) {
92
+
if (PhabricatorEnv::isReadOnly()) {
93
+
return;
94
+
}
95
+
96
+
$table = new self();
97
+
$conn_w = $table->establishConnection('w');
98
+
99
+
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
100
+
101
+
queryfx(
102
+
$conn_w,
103
+
'DELETE FROM %T WHERE cacheIndex = %s',
104
+
$table->getTableName(),
105
+
PhabricatorHash::digestForIndex($key));
106
+
107
+
unset($unguarded);
108
+
}
109
+
110
+
}
+23
-1
src/applications/settings/editor/PhabricatorUserPreferencesEditor.php
+23
-1
src/applications/settings/editor/PhabricatorUserPreferencesEditor.php
···
1
1
<?php
2
2
3
3
final class PhabricatorUserPreferencesEditor
4
-
extends AlmanacEditor {
4
+
extends PhabricatorApplicationTransactionEditor {
5
+
6
+
public function getEditorApplicationClass() {
7
+
return 'PhabricatorSettingsApplication';
8
+
}
5
9
6
10
public function getEditorObjectsDescription() {
7
11
return pht('Settings');
···
127
131
}
128
132
129
133
return $errors;
134
+
}
135
+
136
+
protected function applyFinalEffects(
137
+
PhabricatorLiskDAO $object,
138
+
array $xactions) {
139
+
140
+
$user_phid = $object->getUserPHID();
141
+
if ($user_phid) {
142
+
PhabricatorUserCache::clearCache(
143
+
PhabricatorUserPreferencesCacheType::KEY_PREFERENCES,
144
+
$user_phid);
145
+
} else {
146
+
PhabricatorUserCache::clearCacheForAllUsers(
147
+
PhabricatorUserPreferencesCacheType::KEY_PREFERENCES);
148
+
}
149
+
150
+
151
+
return $xactions;
130
152
}
131
153
132
154
}
+11
src/applications/settings/storage/PhabricatorUserPreferences.php
+11
src/applications/settings/storage/PhabricatorUserPreferences.php
···
165
165
return false;
166
166
}
167
167
168
+
// TODO: Remove this once all edits go through the Editor. For now, some
169
+
// old edits just do direct saves so make sure we nuke the cache.
170
+
public function save() {
171
+
PhabricatorUserCache::clearCache(
172
+
PhabricatorUserPreferencesCacheType::KEY_PREFERENCES,
173
+
$this->getUserPHID());
174
+
175
+
return parent::save();
176
+
}
177
+
178
+
168
179
169
180
/* -( PhabricatorPolicyInterface )----------------------------------------- */
170
181