@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Allow index extensions to skip indexing if the object has not changed
Summary:
Fixes T9890. This allows IndexExtensions to emit an object version.
Before we build indexes, we check if the indexed version is the same as the current version. If it is, we just don't call that extension.
T9890 has a case where this is useful: a script went crazy and posted thousands of comments to a single task.
Without versioning, that results in the same comments being indexed over and over again. With versioning, most of the queue could just exit without doing any work.
Test Plan:
- Added a `sleep(1)` to the actual indexing, used `bin/search index --background` to queue up a lot of tasks, ran them with `bin/phd debug task`, saw them complete very quickly with only one actual index operation performed.
- Used `bin/search index --trace` and `bin/search index --trace --background` to observe the behavior of queries against the index version store, which looked sensible.
- Made comments/transactions, saw versions update.
- Used `bin/remove destroy`, verified index versions were purged.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T9890
Differential Revision: https://secure.phabricator.com/D14845
+175
-3
+7
resources/sql/autopatches/20151221.search.1.version.sql
+7
resources/sql/autopatches/20151221.search.1.version.sql
···
1
+
CREATE TABLE {$NAMESPACE}_search.search_indexversion (
2
+
id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
3
+
objectPHID VARBINARY(64) NOT NULL,
4
+
extensionKey VARCHAR(64) NOT NULL COLLATE {$COLLATE_TEXT},
5
+
version VARCHAR(128) NOT NULL COLLATE {$COLLATE_TEXT},
6
+
UNIQUE KEY `key_object` (objectPHID, extensionKey)
7
+
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
3042
3042
'PhabricatorSearchEngineTestCase' => 'applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php',
3043
3043
'PhabricatorSearchField' => 'applications/search/field/PhabricatorSearchField.php',
3044
3044
'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php',
3045
+
'PhabricatorSearchIndexVersion' => 'applications/search/storage/PhabricatorSearchIndexVersion.php',
3046
+
'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchIndexVersionDestructionEngineExtension.php',
3045
3047
'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php',
3046
3048
'PhabricatorSearchManagementInitWorkflow' => 'applications/search/management/PhabricatorSearchManagementInitWorkflow.php',
3047
3049
'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php',
···
7407
7409
'PhabricatorSearchEngineTestCase' => 'PhabricatorTestCase',
7408
7410
'PhabricatorSearchField' => 'Phobject',
7409
7411
'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController',
7412
+
'PhabricatorSearchIndexVersion' => 'PhabricatorSearchDAO',
7413
+
'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension',
7410
7414
'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow',
7411
7415
'PhabricatorSearchManagementInitWorkflow' => 'PhabricatorSearchManagementWorkflow',
7412
7416
'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow',
+59
src/applications/search/engineextension/PhabricatorFulltextIndexEngineExtension.php
+59
src/applications/search/engineextension/PhabricatorFulltextIndexEngineExtension.php
···
9
9
return pht('Fulltext Engine');
10
10
}
11
11
12
+
public function getIndexVersion($object) {
13
+
$version = array();
14
+
15
+
if ($object instanceof PhabricatorApplicationTransactionInterface) {
16
+
// If this is a normal object with transactions, we only need to
17
+
// reindex it if there are new transactions (or comment edits).
18
+
$version[] = $this->getTransactionVersion($object);
19
+
$version[] = $this->getCommentVersion($object);
20
+
}
21
+
22
+
if (!$version) {
23
+
return null;
24
+
}
25
+
26
+
return implode(':', $version);
27
+
}
28
+
12
29
public function shouldIndexObject($object) {
13
30
return ($object instanceof PhabricatorFulltextInterface);
14
31
}
···
26
43
27
44
$engine->buildFulltextIndexes();
28
45
}
46
+
47
+
private function getTransactionVersion($object) {
48
+
$xaction = $object->getApplicationTransactionTemplate();
49
+
50
+
$xaction_row = queryfx_one(
51
+
$xaction->establishConnection('r'),
52
+
'SELECT id FROM %T WHERE objectPHID = %s
53
+
ORDER BY id DESC LIMIT 1',
54
+
$xaction->getTableName(),
55
+
$object->getPHID());
56
+
if (!$xaction_row) {
57
+
return 'none';
58
+
}
59
+
60
+
return $xaction_row['id'];
61
+
}
62
+
63
+
private function getCommentVersion($object) {
64
+
$xaction = $object->getApplicationTransactionTemplate();
65
+
66
+
try {
67
+
$comment = $xaction->getApplicationTransactionCommentObject();
68
+
} catch (Exception $ex) {
69
+
return 'none';
70
+
}
71
+
72
+
$comment_row = queryfx_one(
73
+
$comment->establishConnection('r'),
74
+
'SELECT c.id FROM %T x JOIN %T c
75
+
ON x.phid = c.transactionPHID
76
+
WHERE x.objectPHID = %s
77
+
ORDER BY c.id DESC LIMIT 1',
78
+
$xaction->getTableName(),
79
+
$comment->getTableName(),
80
+
$object->getPHID());
81
+
if (!$comment_row) {
82
+
return 'none';
83
+
}
84
+
85
+
return $comment_row['id'];
86
+
}
87
+
29
88
30
89
}
+25
src/applications/search/engineextension/PhabricatorSearchIndexVersionDestructionEngineExtension.php
+25
src/applications/search/engineextension/PhabricatorSearchIndexVersionDestructionEngineExtension.php
···
1
+
<?php
2
+
3
+
final class PhabricatorSearchIndexVersionDestructionEngineExtension
4
+
extends PhabricatorDestructionEngineExtension {
5
+
6
+
const EXTENSIONKEY = 'search.index.version';
7
+
8
+
public function getExtensionName() {
9
+
return pht('Search Index Versions');
10
+
}
11
+
12
+
public function destroyObject(
13
+
PhabricatorDestructionEngine $engine,
14
+
$object) {
15
+
16
+
$table = new PhabricatorSearchIndexVersion();
17
+
18
+
queryfx(
19
+
$table->establishConnection('w'),
20
+
'DELETE FROM %T WHERE objectPHID = %s',
21
+
$table->getTableName(),
22
+
$object->getPHID());
23
+
}
24
+
25
+
}
+54
-3
src/applications/search/index/PhabricatorIndexEngine.php
+54
-3
src/applications/search/index/PhabricatorIndexEngine.php
···
45
45
if (idx($parameters, 'force')) {
46
46
$current_versions = array();
47
47
} else {
48
-
// TODO: Load current indexed versions.
49
-
$current_versions = array();
48
+
$keys = array_keys($versions);
49
+
$current_versions = $this->loadIndexVersions($keys);
50
50
}
51
51
52
52
foreach ($versions as $key => $version) {
···
78
78
$extension->indexObject($this, $object);
79
79
}
80
80
81
-
// TODO: Save new index versions.
81
+
$this->saveIndexVersions($this->versions);
82
82
83
83
return $this;
84
84
}
···
94
94
}
95
95
96
96
return $extensions;
97
+
}
98
+
99
+
private function loadIndexVersions(array $extension_keys) {
100
+
if (!$extension_keys) {
101
+
return array();
102
+
}
103
+
104
+
$object = $this->getObject();
105
+
$object_phid = $object->getPHID();
106
+
107
+
$table = new PhabricatorSearchIndexVersion();
108
+
$conn_r = $table->establishConnection('w');
109
+
110
+
$rows = queryfx_all(
111
+
$conn_r,
112
+
'SELECT * FROM %T WHERE objectPHID = %s AND extensionKey IN (%Ls)',
113
+
$table->getTableName(),
114
+
$object_phid,
115
+
$extension_keys);
116
+
117
+
return ipull($rows, 'version', 'extensionKey');
118
+
}
119
+
120
+
private function saveIndexVersions(array $versions) {
121
+
if (!$versions) {
122
+
return;
123
+
}
124
+
125
+
$object = $this->getObject();
126
+
$object_phid = $object->getPHID();
127
+
128
+
$table = new PhabricatorSearchIndexVersion();
129
+
$conn_w = $table->establishConnection('w');
130
+
131
+
$sql = array();
132
+
foreach ($versions as $key => $version) {
133
+
$sql[] = qsprintf(
134
+
$conn_w,
135
+
'(%s, %s, %s)',
136
+
$object_phid,
137
+
$key,
138
+
$version);
139
+
}
140
+
141
+
queryfx(
142
+
$conn_w,
143
+
'INSERT INTO %T (objectPHID, extensionKey, version)
144
+
VALUES %Q
145
+
ON DUPLICATE KEY UPDATE version = VALUES(version)',
146
+
$table->getTableName(),
147
+
implode(', ', $sql));
97
148
}
98
149
99
150
}
+26
src/applications/search/storage/PhabricatorSearchIndexVersion.php
+26
src/applications/search/storage/PhabricatorSearchIndexVersion.php
···
1
+
<?php
2
+
3
+
final class PhabricatorSearchIndexVersion
4
+
extends PhabricatorSearchDAO {
5
+
6
+
protected $objectPHID;
7
+
protected $extensionKey;
8
+
protected $version;
9
+
10
+
protected function getConfiguration() {
11
+
return array(
12
+
self::CONFIG_TIMESTAMPS => false,
13
+
self::CONFIG_COLUMN_SCHEMA => array(
14
+
'extensionKey' => 'text64',
15
+
'version' => 'text128',
16
+
),
17
+
self::CONFIG_KEY_SCHEMA => array(
18
+
'key_object' => array(
19
+
'columns' => array('objectPHID', 'extensionKey'),
20
+
'unique' => true,
21
+
),
22
+
),
23
+
) + parent::getConfiguration();
24
+
}
25
+
26
+
}