@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Implement a more compact, general database-backed key-value cache
Summary:
See discussion in D4204. Facebook currently has a 314MB remarkup cache with a 55MB index, which is slow to access. Under the theory that this is an index size/quality problem (the current index is on a potentially-384-byte field, with many keys sharing prefixes), provide a more general index with fancy new features:
- It implements PhutilKeyValueCache, so it can be a component in cache stacks and supports TTL.
- It has a 12-byte hash-based key.
- It automatically compresses large blocks of data (most of what we store is highly-compressible HTML).
Test Plan:
- Basics:
- Loaded /paste/, saw caches generate and save.
- Reloaded /paste/, saw the page hit cache.
- GC:
- Ran GC daemon, saw nothing.
- Set maximum lifetime to 1 second, ran GC daemon, saw it collect the entire cache.
- Deflate:
- Selected row formats from the database, saw a mixture of 'raw' and 'deflate' storage.
- Used profiler to verify that 'deflate' is fast (12 calls @ 220us on my paste list).
- Ran unit tests
Reviewers: vrana, btrahan
Reviewed By: vrana
CC: aran
Differential Revision: https://secure.phabricator.com/D4259
+407
-13
+8
conf/default.conf.php
+8
conf/default.conf.php
···
1130
1130
'remarkup.enable-embedded-youtube' => false,
1131
1131
1132
1132
1133
+
// -- Cache ----------------------------------------------------------------- //
1134
+
1135
+
// Set this to false to disable the use of gzdeflate()-based compression in
1136
+
// some caches. This may give you less performant (but more debuggable)
1137
+
// caching.
1138
+
'cache.enable-deflate' => true,
1139
+
1133
1140
// -- Garbage Collection ---------------------------------------------------- //
1134
1141
1135
1142
// Phabricator generates various logs and caches in the database which can
···
1160
1167
'gcdaemon.ttl.differential-parse-cache' => 14 * (24 * 60 * 60),
1161
1168
'gcdaemon.ttl.markup-cache' => 30 * (24 * 60 * 60),
1162
1169
'gcdaemon.ttl.task-archive' => 14 * (24 * 60 * 60),
1170
+
'gcdaemon.ttl.general-cache' => 30 * (24 * 60 * 60),
1163
1171
1164
1172
1165
1173
// -- Feed ------------------------------------------------------------------ //
+11
resources/sql/patches/20121220.generalcache.sql
+11
resources/sql/patches/20121220.generalcache.sql
···
1
+
CREATE TABLE {$NAMESPACE}_cache.cache_general (
2
+
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
3
+
cacheKeyHash CHAR(12) BINARY NOT NULL,
4
+
cacheKey VARCHAR(128) NOT NULL COLLATE utf8_bin,
5
+
cacheFormat VARCHAR(16) NOT NULL COLLATE utf8_bin,
6
+
cacheData LONGBLOB NOT NULL,
7
+
cacheCreated INT UNSIGNED NOT NULL,
8
+
cacheExpires INT UNSIGNED,
9
+
KEY `key_cacheCreated` (cacheCreated),
10
+
UNIQUE KEY `key_cacheKeyHash` (cacheKeyHash)
11
+
) ENGINE=InnoDB, COLLATE utf8_general_ci;
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
505
505
'JavelinUIExample' => 'applications/uiexample/examples/JavelinUIExample.php',
506
506
'JavelinViewExample' => 'applications/uiexample/examples/JavelinViewExample.php',
507
507
'JavelinViewExampleServerView' => 'applications/uiexample/examples/JavelinViewExampleServerView.php',
508
+
'LiskChunkTestCase' => 'infrastructure/storage/lisk/__tests__/LiskChunkTestCase.php',
508
509
'LiskDAO' => 'infrastructure/storage/lisk/LiskDAO.php',
509
510
'LiskDAOSet' => 'infrastructure/storage/lisk/LiskDAOSet.php',
510
511
'LiskDAOTestCase' => 'infrastructure/storage/lisk/__tests__/LiskDAOTestCase.php',
···
835
836
'PhabricatorInlineSummaryView' => 'infrastructure/diff/view/PhabricatorInlineSummaryView.php',
836
837
'PhabricatorJavelinLinter' => 'infrastructure/lint/linter/PhabricatorJavelinLinter.php',
837
838
'PhabricatorJumpNavHandler' => 'applications/search/engine/PhabricatorJumpNavHandler.php',
839
+
'PhabricatorKeyValueDatabaseCache' => 'applications/cache/PhabricatorKeyValueDatabaseCache.php',
838
840
'PhabricatorLDAPLoginController' => 'applications/auth/controller/PhabricatorLDAPLoginController.php',
839
841
'PhabricatorLDAPProvider' => 'applications/auth/ldap/PhabricatorLDAPProvider.php',
840
842
'PhabricatorLDAPRegistrationController' => 'applications/auth/controller/PhabricatorLDAPRegistrationController.php',
···
1789
1791
'JavelinUIExample' => 'PhabricatorUIExample',
1790
1792
'JavelinViewExample' => 'PhabricatorUIExample',
1791
1793
'JavelinViewExampleServerView' => 'AphrontView',
1794
+
'LiskChunkTestCase' => 'PhabricatorTestCase',
1792
1795
'LiskDAOTestCase' => 'PhabricatorTestCase',
1793
1796
'LiskEphemeralObjectException' => 'Exception',
1794
1797
'LiskFixtureTestCase' => 'PhabricatorTestCase',
···
2118
2121
'PhabricatorInlineCommentPreviewController' => 'PhabricatorController',
2119
2122
'PhabricatorInlineSummaryView' => 'AphrontView',
2120
2123
'PhabricatorJavelinLinter' => 'ArcanistLinter',
2124
+
'PhabricatorKeyValueDatabaseCache' => 'PhutilKeyValueCache',
2121
2125
'PhabricatorLDAPLoginController' => 'PhabricatorAuthController',
2122
2126
'PhabricatorLDAPRegistrationController' => 'PhabricatorAuthController',
2123
2127
'PhabricatorLDAPUnknownUserException' => 'Exception',
+232
src/applications/cache/PhabricatorKeyValueDatabaseCache.php
+232
src/applications/cache/PhabricatorKeyValueDatabaseCache.php
···
1
+
<?php
2
+
3
+
final class PhabricatorKeyValueDatabaseCache
4
+
extends PhutilKeyValueCache {
5
+
6
+
const CACHE_FORMAT_RAW = 'raw';
7
+
const CACHE_FORMAT_DEFLATE = 'deflate';
8
+
9
+
public function setKeys(array $keys, $ttl = null) {
10
+
$call_id = null;
11
+
if ($this->getProfiler()) {
12
+
$call_id = $this->getProfiler()->beginServiceCall(
13
+
array(
14
+
'type' => 'kvcache-set',
15
+
'name' => 'phabricator-db',
16
+
'keys' => array_keys($keys),
17
+
'ttl' => $ttl,
18
+
));
19
+
}
20
+
21
+
if ($keys) {
22
+
$map = $this->digestKeys(array_keys($keys));
23
+
$conn_w = $this->establishConnection('w');
24
+
25
+
$sql = array();
26
+
foreach ($map as $key => $hash) {
27
+
$value = $keys[$key];
28
+
29
+
list($format, $storage_value) = $this->willWriteValue($key, $value);
30
+
31
+
$sql[] = qsprintf(
32
+
$conn_w,
33
+
'(%s, %s, %s, %s, %d, %nd)',
34
+
$hash,
35
+
$key,
36
+
$format,
37
+
$storage_value,
38
+
time(),
39
+
$ttl ? (time() + $ttl) : null);
40
+
}
41
+
42
+
$guard = AphrontWriteGuard::beginScopedUnguardedWrites();
43
+
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
44
+
queryfx(
45
+
$conn_w,
46
+
'INSERT INTO %T
47
+
(cacheKeyHash, cacheKey, cacheFormat, cacheData,
48
+
cacheCreated, cacheExpires) VALUES %Q
49
+
ON DUPLICATE KEY UPDATE
50
+
cacheKey = VALUES(cacheKey),
51
+
cacheFormat = VALUES(cacheFormat),
52
+
cacheData = VALUES(cacheData),
53
+
cacheCreated = VALUES(cacheCreated),
54
+
cacheExpires = VALUES(cacheExpires)',
55
+
$this->getTableName(),
56
+
$chunk);
57
+
}
58
+
unset($guard);
59
+
}
60
+
61
+
if ($call_id) {
62
+
$this->getProfiler()->endServiceCall($call_id, array());
63
+
}
64
+
65
+
return $this;
66
+
}
67
+
68
+
public function getKeys(array $keys) {
69
+
$call_id = null;
70
+
if ($this->getProfiler()) {
71
+
$call_id = $this->getProfiler()->beginServiceCall(
72
+
array(
73
+
'type' => 'kvcache-get',
74
+
'name' => 'phabricator-db',
75
+
'keys' => $keys,
76
+
));
77
+
}
78
+
79
+
$results = array();
80
+
if ($keys) {
81
+
$map = $this->digestKeys($keys);
82
+
83
+
$rows = queryfx_all(
84
+
$this->establishConnection('r'),
85
+
'SELECT * FROM %T WHERE cacheKeyHash IN (%Ls)',
86
+
$this->getTableName(),
87
+
$map);
88
+
$rows = ipull($rows, null, 'cacheKey');
89
+
90
+
foreach ($keys as $key) {
91
+
if (empty($rows[$key])) {
92
+
continue;
93
+
}
94
+
95
+
$row = $rows[$key];
96
+
97
+
if ($row['cacheExpires'] && ($row['cacheExpires'] < time())) {
98
+
continue;
99
+
}
100
+
101
+
try {
102
+
$results[$key] = $this->didReadValue(
103
+
$row['cacheFormat'],
104
+
$row['cacheData']);
105
+
} catch (Exception $ex) {
106
+
// Treat this as a cache miss.
107
+
phlog($ex);
108
+
}
109
+
}
110
+
}
111
+
112
+
if ($call_id) {
113
+
$this->getProfiler()->endServiceCall(
114
+
$call_id,
115
+
array(
116
+
'hits' => array_keys($results),
117
+
));
118
+
}
119
+
120
+
return $results;
121
+
}
122
+
123
+
public function deleteKeys(array $keys) {
124
+
$call_id = null;
125
+
if ($this->getProfiler()) {
126
+
$call_id = $this->getProfiler()->beginServiceCall(
127
+
array(
128
+
'type' => 'kvcache-del',
129
+
'name' => 'phabricator-db',
130
+
'keys' => $keys,
131
+
));
132
+
}
133
+
134
+
if ($keys) {
135
+
$map = $this->digestKeys($keys);
136
+
queryfx(
137
+
$this->establishConnection('w'),
138
+
'DELETE FROM %T WHERE cacheKeyHash IN (%Ls)',
139
+
$this->getTableName(),
140
+
$keys);
141
+
}
142
+
143
+
if ($call_id) {
144
+
$this->getProfiler()->endServiceCall($call_id, array());
145
+
}
146
+
147
+
return $this;
148
+
}
149
+
150
+
public function destroyCache() {
151
+
queryfx(
152
+
$this->establishConnection('w'),
153
+
'DELETE FROM %T',
154
+
$this->getTableName());
155
+
return $this;
156
+
}
157
+
158
+
159
+
/* -( Raw Cache Access )--------------------------------------------------- */
160
+
161
+
162
+
public function establishConnection($mode) {
163
+
// TODO: This is the only concrete table we have on the database right
164
+
// now.
165
+
return id(new PhabricatorMarkupCache())->establishConnection($mode);
166
+
}
167
+
168
+
public function getTableName() {
169
+
return 'cache_general';
170
+
}
171
+
172
+
173
+
/* -( Implementation )----------------------------------------------------- */
174
+
175
+
176
+
private function digestKeys(array $keys) {
177
+
$map = array();
178
+
foreach ($keys as $key) {
179
+
$map[$key] = PhabricatorHash::digestForIndex($key);
180
+
}
181
+
return $map;
182
+
}
183
+
184
+
private function willWriteValue($key, $value) {
185
+
if (!is_string($value)) {
186
+
throw new Exception("Only strings may be written to the DB cache!");
187
+
}
188
+
189
+
static $can_deflate;
190
+
if ($can_deflate === null) {
191
+
$can_deflate = function_exists('gzdeflate') &&
192
+
PhabricatorEnv::getEnvConfig('cache.enable-deflate');
193
+
}
194
+
195
+
// If the value is larger than 1KB, we have gzdeflate(), we successfully
196
+
// can deflate it, and it benefits from deflation, store it deflated.
197
+
if ($can_deflate) {
198
+
$len = strlen($value);
199
+
if ($len > 1024) {
200
+
$deflated = gzdeflate($value);
201
+
if ($deflated !== false) {
202
+
$deflated_len = strlen($deflated);
203
+
if ($deflated_len < ($len / 2)) {
204
+
return array(self::CACHE_FORMAT_DEFLATE, $deflated);
205
+
}
206
+
}
207
+
}
208
+
}
209
+
210
+
return array(self::CACHE_FORMAT_RAW, $value);
211
+
}
212
+
213
+
private function didReadValue($format, $value) {
214
+
switch ($format) {
215
+
case self::CACHE_FORMAT_RAW:
216
+
return $value;
217
+
case self::CACHE_FORMAT_DEFLATE:
218
+
if (!function_exists('gzinflate')) {
219
+
throw new Exception("No gzinflate() to read deflated cache.");
220
+
}
221
+
$value = gzinflate($value);
222
+
if ($value === false) {
223
+
throw new Exception("Failed to deflate cache.");
224
+
}
225
+
return $value;
226
+
default:
227
+
throw new Exception("Unknown cache format.");
228
+
}
229
+
}
230
+
231
+
232
+
}
+11
-13
src/applications/paste/query/PhabricatorPasteQuery.php
+11
-13
src/applications/paste/query/PhabricatorPasteQuery.php
···
124
124
}
125
125
126
126
private function loadContent(array $pastes) {
127
+
$cache = id(new PhabricatorKeyValueDatabaseCache())
128
+
->setProfiler(PhutilServiceProfiler::getInstance());
129
+
127
130
$keys = array();
128
131
foreach ($pastes as $paste) {
129
132
$keys[] = $this->getContentCacheKey($paste);
130
133
}
131
134
132
-
// TODO: Move to a more appropriate/general cache once we have one? For
133
-
// now, this gets automatic GC.
134
-
$caches = id(new PhabricatorMarkupCache())->loadAllWhere(
135
-
'cacheKey IN (%Ls)',
136
-
$keys);
137
-
$caches = mpull($caches, null, 'getCacheKey');
135
+
136
+
$caches = $cache->getKeys($keys);
138
137
139
138
$need_raw = array();
140
139
foreach ($pastes as $paste) {
141
140
$key = $this->getContentCacheKey($paste);
142
141
if (isset($caches[$key])) {
143
-
$paste->attachContent($caches[$key]->getCacheData());
142
+
$paste->attachContent($caches[$key]);
144
143
} else {
145
144
$need_raw[] = $paste;
146
145
}
···
150
149
return;
151
150
}
152
151
152
+
$write_data = array();
153
+
153
154
$this->loadRawContent($need_raw);
154
155
foreach ($need_raw as $paste) {
155
156
$content = $this->buildContent($paste);
156
157
$paste->attachContent($content);
157
158
158
-
$guard = AphrontWriteGuard::beginScopedUnguardedWrites();
159
-
id(new PhabricatorMarkupCache())
160
-
->setCacheKey($this->getContentCacheKey($paste))
161
-
->setCacheData($content)
162
-
->replace();
163
-
unset($guard);
159
+
$write_data[$this->getContentCacheKey($paste)] = (string)$content;
164
160
}
161
+
162
+
$cache->setKeys($write_data);
165
163
}
166
164
167
165
+23
src/infrastructure/daemon/PhabricatorGarbageCollectorDaemon.php
+23
src/infrastructure/daemon/PhabricatorGarbageCollectorDaemon.php
···
49
49
$n_parse = $this->collectParseCaches();
50
50
$n_markup = $this->collectMarkupCaches();
51
51
$n_tasks = $this->collectArchivedTasks();
52
+
$n_cache = $this->collectGeneralCaches();
52
53
53
54
$collected = array(
54
55
'Herald Transcript' => $n_herald,
···
56
57
'Differential Parse Cache' => $n_parse,
57
58
'Markup Cache' => $n_markup,
58
59
'Archived Tasks' => $n_tasks,
60
+
'General Cache Entries' => $n_cache,
59
61
);
60
62
$collected = array_filter($collected);
61
63
···
198
200
$table->saveTransaction();
199
201
200
202
return count($task_ids);
203
+
}
204
+
205
+
206
+
private function collectGeneralCaches() {
207
+
$key = 'gcdaemon.ttl.general-cache';
208
+
$ttl = PhabricatorEnv::getEnvConfig($key);
209
+
if ($ttl <= 0) {
210
+
return 0;
211
+
}
212
+
213
+
$cache = new PhabricatorKeyValueDatabaseCache();
214
+
$conn_w = $cache->establishConnection('w');
215
+
216
+
queryfx(
217
+
$conn_w,
218
+
'DELETE FROM %T WHERE cacheCreated < %d
219
+
ORDER BY cacheCreated ASC LIMIT 100',
220
+
$cache->getTableName(),
221
+
time() - $ttl);
222
+
223
+
return $conn_w->getAffectedRows();
201
224
}
202
225
203
226
}
+59
src/infrastructure/storage/lisk/PhabricatorLiskDAO.php
+59
src/infrastructure/storage/lisk/PhabricatorLiskDAO.php
···
148
148
protected function getConnectionNamespace() {
149
149
return self::getStorageNamespace().'_'.$this->getApplicationName();
150
150
}
151
+
152
+
153
+
/**
154
+
* Break a list of escaped SQL statement fragments (e.g., VALUES lists for
155
+
* INSERT, previously built with @{function:qsprintf}) into chunks which will
156
+
* fit under the MySQL 'max_allowed_packet' limit.
157
+
*
158
+
* Chunks are glued together with `$glue`, by default ", ".
159
+
*
160
+
* If a statement is too large to fit within the limit, it is broken into
161
+
* its own chunk (but might fail when the query executes).
162
+
*/
163
+
public static function chunkSQL(
164
+
array $fragments,
165
+
$glue = ', ',
166
+
$limit = null) {
167
+
168
+
if ($limit === null) {
169
+
// NOTE: Hard-code this at 1MB for now, minus a 10% safety buffer.
170
+
// Eventually we could query MySQL or let the user configure it.
171
+
$limit = (int)((1024 * 1024) * 0.90);
172
+
}
173
+
174
+
$result = array();
175
+
176
+
$chunk = array();
177
+
$len = 0;
178
+
$glue_len = strlen($glue);
179
+
foreach ($fragments as $fragment) {
180
+
$this_len = strlen($fragment);
181
+
182
+
if ($chunk) {
183
+
// Chunks after the first also imply glue.
184
+
$this_len += $glue_len;
185
+
}
186
+
187
+
if ($len + $this_len <= $limit) {
188
+
$len += $this_len;
189
+
$chunk[] = $fragment;
190
+
} else {
191
+
if ($chunk) {
192
+
$result[] = $chunk;
193
+
}
194
+
$len = strlen($fragment);
195
+
$chunk = array($fragment);
196
+
}
197
+
}
198
+
199
+
if ($chunk) {
200
+
$result[] = $chunk;
201
+
}
202
+
203
+
foreach ($result as $key => $fragment_list) {
204
+
$result[$key] = implode($glue, $fragment_list);
205
+
}
206
+
207
+
return $result;
208
+
}
209
+
151
210
}
+55
src/infrastructure/storage/lisk/__tests__/LiskChunkTestCase.php
+55
src/infrastructure/storage/lisk/__tests__/LiskChunkTestCase.php
···
1
+
<?php
2
+
3
+
final class LiskChunkTestCase extends PhabricatorTestCase {
4
+
5
+
public function testSQLChunking() {
6
+
$fragments = array(
7
+
'a', 'a',
8
+
'b', 'b',
9
+
'ccc',
10
+
'dd',
11
+
'e',
12
+
);
13
+
14
+
$this->assertEqual(
15
+
array(
16
+
'aa',
17
+
'bb',
18
+
'ccc',
19
+
'dd',
20
+
'e',
21
+
),
22
+
PhabricatorLiskDAO::chunkSQL($fragments, '', 2));
23
+
24
+
25
+
$fragments = array(
26
+
'a', 'a', 'a', 'XX', 'a', 'a', 'a', 'a'
27
+
);
28
+
29
+
$this->assertEqual(
30
+
array(
31
+
'a, a, a',
32
+
'XX, a, a',
33
+
'a, a',
34
+
),
35
+
PhabricatorLiskDAO::chunkSQL($fragments, ', ', 8));
36
+
37
+
38
+
$fragments = array(
39
+
'xxxxxxxxxx',
40
+
'yyyyyyyyyy',
41
+
'a', 'b', 'c',
42
+
'zzzzzzzzzz',
43
+
);
44
+
45
+
$this->assertEqual(
46
+
array(
47
+
'xxxxxxxxxx',
48
+
'yyyyyyyyyy',
49
+
'a, b, c',
50
+
'zzzzzzzzzz',
51
+
),
52
+
PhabricatorLiskDAO::chunkSQL($fragments, ', ', 8));
53
+
}
54
+
55
+
}
+4
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php
+4
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php