@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Lift Conpherence indexing up out of the Fulltext index
Summary:
Ref T9979. There are currently some hacks around Conpherence indexing: it does not really use the fulltext index, but its own specialized index. However, it's kind of hacked up so it can get reindexed by the normal indexing pipeline.
Lift it up into IndexEngine, instead of FulltextEngine. Specifically, the new stuff is going to look like this:
- IndexEngine: Rebuild all indexes.
- ConpherenceIndexExtension: Rebuild thread indexes.
- ProjectMemberIndexExtension: Rebuild project membership views.
- NgramIndexExtension: Rebuild ngram indexes.
- FulltextIndexExtension / FulltextEngine: Rebuild fulltext indexes, a special type of index.
- FulltextCommentExtension: Rebuild comment fulltext indexes.
- FulltextProjectExtension: Rebuild project fulltext indexes.
- etc.
Most of this is at least sort-of-in-place as of this diff, although some of the part in the middle is still pretty rough.
Test Plan:
- Made a unique comment in a Conpherence thread.
- Used `bin/search index --force` to rebuild the index.
- Searched for the comment.
- Found the thread.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T9979
Differential Revision: https://secure.phabricator.com/D14841
+316
-140
+6
-2
src/__phutil_library_map__.php
+6
-2
src/__phutil_library_map__.php
···
285
285
'ConpherenceSettings' => 'applications/conpherence/constants/ConpherenceSettings.php',
286
286
'ConpherenceTestCase' => 'applications/conpherence/__tests__/ConpherenceTestCase.php',
287
287
'ConpherenceThread' => 'applications/conpherence/storage/ConpherenceThread.php',
288
-
'ConpherenceThreadIndexer' => 'applications/conpherence/search/ConpherenceThreadIndexer.php',
288
+
'ConpherenceThreadIndexEngineExtension' => 'applications/conpherence/engineextension/ConpherenceThreadIndexEngineExtension.php',
289
289
'ConpherenceThreadListView' => 'applications/conpherence/view/ConpherenceThreadListView.php',
290
290
'ConpherenceThreadMailReceiver' => 'applications/conpherence/mail/ConpherenceThreadMailReceiver.php',
291
291
'ConpherenceThreadMembersPolicyRule' => 'applications/conpherence/policyrule/ConpherenceThreadMembersPolicyRule.php',
···
2378
2378
'PhabricatorImageTransformer' => 'applications/files/PhabricatorImageTransformer.php',
2379
2379
'PhabricatorImagemagickSetupCheck' => 'applications/config/check/PhabricatorImagemagickSetupCheck.php',
2380
2380
'PhabricatorIndexEngine' => 'applications/search/index/PhabricatorIndexEngine.php',
2381
+
'PhabricatorIndexEngineExtension' => 'applications/search/index/PhabricatorIndexEngineExtension.php',
2382
+
'PhabricatorIndexEngineExtensionModule' => 'applications/search/index/PhabricatorIndexEngineExtensionModule.php',
2381
2383
'PhabricatorInfrastructureTestCase' => '__tests__/PhabricatorInfrastructureTestCase.php',
2382
2384
'PhabricatorInlineCommentController' => 'infrastructure/diff/PhabricatorInlineCommentController.php',
2383
2385
'PhabricatorInlineCommentInterface' => 'infrastructure/diff/interface/PhabricatorInlineCommentInterface.php',
···
4181
4183
'PhabricatorMentionableInterface',
4182
4184
'PhabricatorDestructibleInterface',
4183
4185
),
4184
-
'ConpherenceThreadIndexer' => 'PhabricatorSearchDocumentIndexer',
4186
+
'ConpherenceThreadIndexEngineExtension' => 'PhabricatorIndexEngineExtension',
4185
4187
'ConpherenceThreadListView' => 'AphrontView',
4186
4188
'ConpherenceThreadMailReceiver' => 'PhabricatorObjectMailReceiver',
4187
4189
'ConpherenceThreadMembersPolicyRule' => 'PhabricatorPolicyRule',
···
6606
6608
'PhabricatorImageTransformer' => 'Phobject',
6607
6609
'PhabricatorImagemagickSetupCheck' => 'PhabricatorSetupCheck',
6608
6610
'PhabricatorIndexEngine' => 'Phobject',
6611
+
'PhabricatorIndexEngineExtension' => 'Phobject',
6612
+
'PhabricatorIndexEngineExtensionModule' => 'PhabricatorConfigModule',
6609
6613
'PhabricatorInfrastructureTestCase' => 'PhabricatorTestCase',
6610
6614
'PhabricatorInlineCommentController' => 'PhabricatorController',
6611
6615
'PhabricatorInlineCommentInterface' => 'PhabricatorMarkupInterface',
-16
src/applications/conpherence/editor/ConpherenceEditor.php
-16
src/applications/conpherence/editor/ConpherenceEditor.php
···
601
601
return true;
602
602
}
603
603
604
-
protected function getSearchContextParameter(
605
-
PhabricatorLiskDAO $object,
606
-
array $xactions) {
607
-
608
-
$comment_phids = array();
609
-
foreach ($xactions as $xaction) {
610
-
if ($xaction->hasComment()) {
611
-
$comment_phids[] = $xaction->getPHID();
612
-
}
613
-
}
614
-
615
-
return array(
616
-
'commentPHIDs' => $comment_phids,
617
-
);
618
-
}
619
-
620
604
protected function extractFilePHIDsFromCustomTransaction(
621
605
PhabricatorLiskDAO $object,
622
606
PhabricatorApplicationTransaction $xaction) {
+81
src/applications/conpherence/engineextension/ConpherenceThreadIndexEngineExtension.php
+81
src/applications/conpherence/engineextension/ConpherenceThreadIndexEngineExtension.php
···
1
+
<?php
2
+
3
+
final class ConpherenceThreadIndexEngineExtension
4
+
extends PhabricatorIndexEngineExtension {
5
+
6
+
const EXTENSIONKEY = 'conpherence.thread';
7
+
8
+
public function getExtensionName() {
9
+
return pht('Conpherence Threads');
10
+
}
11
+
12
+
public function shouldIndexObject($object) {
13
+
return ($object instanceof ConpherenceThread);
14
+
}
15
+
16
+
public function indexObject(
17
+
PhabricatorIndexEngine $engine,
18
+
$object) {
19
+
20
+
$force = $this->shouldForceFullReindex();
21
+
22
+
if (!$force) {
23
+
$xaction_phids = $this->getParameter('transactionPHIDs');
24
+
if (!$xaction_phids) {
25
+
return;
26
+
}
27
+
}
28
+
29
+
$query = id(new ConpherenceTransactionQuery())
30
+
->setViewer($this->getViewer())
31
+
->withObjectPHIDs(array($object->getPHID()))
32
+
->withTransactionTypes(array(PhabricatorTransactions::TYPE_COMMENT))
33
+
->needComments(true);
34
+
35
+
if (!$force) {
36
+
$query->withPHIDs($xaction_phids);
37
+
}
38
+
39
+
$xactions = $query->execute();
40
+
41
+
if (!$xactions) {
42
+
return;
43
+
}
44
+
45
+
foreach ($xactions as $xaction) {
46
+
$this->indexComment($object, $xaction);
47
+
}
48
+
}
49
+
50
+
private function indexComment(
51
+
ConpherenceThread $thread,
52
+
ConpherenceTransaction $xaction) {
53
+
54
+
$previous = id(new ConpherenceTransactionQuery())
55
+
->setViewer($this->getViewer())
56
+
->withObjectPHIDs(array($thread->getPHID()))
57
+
->withTransactionTypes(array(PhabricatorTransactions::TYPE_COMMENT))
58
+
->setAfterID($xaction->getID())
59
+
->setLimit(1)
60
+
->executeOne();
61
+
62
+
$index = id(new ConpherenceIndex())
63
+
->setThreadPHID($thread->getPHID())
64
+
->setTransactionPHID($xaction->getPHID())
65
+
->setPreviousTransactionPHID($previous ? $previous->getPHID() : null)
66
+
->setCorpus($xaction->getComment()->getContent());
67
+
68
+
queryfx(
69
+
$index->establishConnection('w'),
70
+
'INSERT INTO %T
71
+
(threadPHID, transactionPHID, previousTransactionPHID, corpus)
72
+
VALUES (%s, %s, %ns, %s)
73
+
ON DUPLICATE KEY UPDATE corpus = VALUES(corpus)',
74
+
$index->getTableName(),
75
+
$index->getThreadPHID(),
76
+
$index->getTransactionPHID(),
77
+
$index->getPreviousTransactionPHID(),
78
+
$index->getCorpus());
79
+
}
80
+
81
+
}
-89
src/applications/conpherence/search/ConpherenceThreadIndexer.php
-89
src/applications/conpherence/search/ConpherenceThreadIndexer.php
···
1
-
<?php
2
-
3
-
final class ConpherenceThreadIndexer
4
-
extends PhabricatorSearchDocumentIndexer {
5
-
6
-
public function getIndexableObject() {
7
-
return new ConpherenceThread();
8
-
}
9
-
10
-
protected function loadDocumentByPHID($phid) {
11
-
$object = id(new ConpherenceThreadQuery())
12
-
->setViewer($this->getViewer())
13
-
->withPHIDs(array($phid))
14
-
->executeOne();
15
-
16
-
if (!$object) {
17
-
throw new Exception(pht('No thread "%s" exists!', $phid));
18
-
}
19
-
20
-
return $object;
21
-
}
22
-
23
-
protected function buildAbstractDocumentByPHID($phid) {
24
-
$thread = $this->loadDocumentByPHID($phid);
25
-
26
-
// NOTE: We're explicitly not building a document here, only rebuilding
27
-
// the Conpherence search index.
28
-
29
-
$context = nonempty($this->getContext(), array());
30
-
$comment_phids = idx($context, 'commentPHIDs');
31
-
32
-
if (is_array($comment_phids) && !$comment_phids) {
33
-
// If this property is set, but empty, the transaction did not
34
-
// include any chat text. For example, a user might have left the
35
-
// conversation.
36
-
return null;
37
-
}
38
-
39
-
$query = id(new ConpherenceTransactionQuery())
40
-
->setViewer($this->getViewer())
41
-
->withObjectPHIDs(array($thread->getPHID()))
42
-
->withTransactionTypes(array(PhabricatorTransactions::TYPE_COMMENT))
43
-
->needComments(true);
44
-
45
-
if ($comment_phids !== null) {
46
-
$query->withPHIDs($comment_phids);
47
-
}
48
-
49
-
$xactions = $query->execute();
50
-
51
-
foreach ($xactions as $xaction) {
52
-
$this->indexComment($thread, $xaction);
53
-
}
54
-
55
-
return null;
56
-
}
57
-
58
-
private function indexComment(
59
-
ConpherenceThread $thread,
60
-
ConpherenceTransaction $xaction) {
61
-
62
-
$previous = id(new ConpherenceTransactionQuery())
63
-
->setViewer($this->getViewer())
64
-
->withObjectPHIDs(array($thread->getPHID()))
65
-
->withTransactionTypes(array(PhabricatorTransactions::TYPE_COMMENT))
66
-
->setAfterID($xaction->getID())
67
-
->setLimit(1)
68
-
->executeOne();
69
-
70
-
$index = id(new ConpherenceIndex())
71
-
->setThreadPHID($thread->getPHID())
72
-
->setTransactionPHID($xaction->getPHID())
73
-
->setPreviousTransactionPHID($previous ? $previous->getPHID() : null)
74
-
->setCorpus($xaction->getComment()->getContent());
75
-
76
-
queryfx(
77
-
$index->establishConnection('w'),
78
-
'INSERT INTO %T
79
-
(threadPHID, transactionPHID, previousTransactionPHID, corpus)
80
-
VALUES (%s, %s, %ns, %s)
81
-
ON DUPLICATE KEY UPDATE corpus = VALUES(corpus)',
82
-
$index->getTableName(),
83
-
$index->getThreadPHID(),
84
-
$index->getTransactionPHID(),
85
-
$index->getPreviousTransactionPHID(),
86
-
$index->getCorpus());
87
-
}
88
-
89
-
}
+96
-2
src/applications/search/index/PhabricatorIndexEngine.php
+96
-2
src/applications/search/index/PhabricatorIndexEngine.php
···
2
2
3
3
final class PhabricatorIndexEngine extends Phobject {
4
4
5
-
public function indexDocumentByPHID($phid, $context) {
5
+
private $object;
6
+
private $extensions;
7
+
private $versions;
8
+
private $parameters;
9
+
10
+
public function setParameters(array $parameters) {
11
+
$this->parameters = $parameters;
12
+
return $this;
13
+
}
14
+
15
+
public function getParameters() {
16
+
return $this->parameters;
17
+
}
18
+
19
+
public function setObject($object) {
20
+
$this->object = $object;
21
+
return $this;
22
+
}
23
+
24
+
public function getObject() {
25
+
return $this->object;
26
+
}
27
+
28
+
public function shouldIndexObject() {
29
+
$extensions = $this->newExtensions();
30
+
31
+
$parameters = $this->getParameters();
32
+
foreach ($extensions as $extension) {
33
+
$extension->setParameters($parameters);
34
+
}
35
+
36
+
$object = $this->getObject();
37
+
$versions = array();
38
+
foreach ($extensions as $key => $extension) {
39
+
$version = $extension->getIndexVersion($object);
40
+
if ($version !== null) {
41
+
$versions[$key] = (string)$version;
42
+
}
43
+
}
44
+
45
+
if (idx($parameters, 'force')) {
46
+
$current_versions = array();
47
+
} else {
48
+
// TODO: Load current indexed versions.
49
+
$current_versions = array();
50
+
}
51
+
52
+
foreach ($versions as $key => $version) {
53
+
$current_version = idx($current_versions, $key);
54
+
55
+
if ($current_version === null) {
56
+
continue;
57
+
}
58
+
59
+
// If nothing has changed since we built the current index, we do not
60
+
// need to rebuild the index.
61
+
if ($current_version === $version) {
62
+
unset($extensions[$key]);
63
+
}
64
+
}
65
+
66
+
$this->extensions = $extensions;
67
+
$this->versions = $versions;
68
+
69
+
// We should index the object only if there is any work to be done.
70
+
return (bool)$this->extensions;
71
+
}
72
+
73
+
public function indexObject() {
74
+
$extensions = $this->extensions;
75
+
$object = $this->getObject();
76
+
77
+
foreach ($extensions as $key => $extension) {
78
+
$extension->indexObject($this, $object);
79
+
}
80
+
81
+
// TODO: Save new index versions.
82
+
83
+
return $this;
84
+
}
85
+
86
+
private function newExtensions() {
87
+
$object = $this->getObject();
88
+
89
+
$extensions = PhabricatorIndexEngineExtension::getAllExtensions();
90
+
foreach ($extensions as $key => $extension) {
91
+
if (!$extension->shouldIndexObject($object)) {
92
+
unset($extensions[$key]);
93
+
}
94
+
}
95
+
96
+
return $extensions;
97
+
}
98
+
99
+
public function indexDocumentByPHID($phid) {
6
100
$indexers = id(new PhutilClassMapQuery())
7
101
->setAncestorClass('PhabricatorSearchDocumentIndexer')
8
102
->execute();
9
103
10
104
foreach ($indexers as $indexer) {
11
105
if ($indexer->shouldIndexDocumentByPHID($phid)) {
12
-
$indexer->indexDocumentByPHID($phid, $context);
106
+
$indexer->indexDocumentByPHID($phid);
13
107
break;
14
108
}
15
109
}
+48
src/applications/search/index/PhabricatorIndexEngineExtension.php
+48
src/applications/search/index/PhabricatorIndexEngineExtension.php
···
1
+
<?php
2
+
3
+
abstract class PhabricatorIndexEngineExtension extends Phobject {
4
+
5
+
private $parameters;
6
+
private $forceFullReindex;
7
+
8
+
public function setParameters(array $parameters) {
9
+
$this->parameters = $parameters;
10
+
return $this;
11
+
}
12
+
13
+
public function getParameter($key, $default = null) {
14
+
return idx($this->parameters, $key, $default);
15
+
}
16
+
17
+
final public function getExtensionKey() {
18
+
return $this->getPhobjectClassConstant('EXTENSIONKEY');
19
+
}
20
+
21
+
final protected function getViewer() {
22
+
return PhabricatorUser::getOmnipotentUser();
23
+
}
24
+
25
+
abstract public function getExtensionName();
26
+
27
+
abstract public function shouldIndexObject($object);
28
+
29
+
abstract public function indexObject(
30
+
PhabricatorIndexEngine $engine,
31
+
$object);
32
+
33
+
public function getIndexVersion($object) {
34
+
return null;
35
+
}
36
+
37
+
final public static function getAllExtensions() {
38
+
return id(new PhutilClassMapQuery())
39
+
->setAncestorClass(__CLASS__)
40
+
->setUniqueMethod('getExtensionKey')
41
+
->execute();
42
+
}
43
+
44
+
final public function shouldForceFullReindex() {
45
+
return $this->getParameter('force');
46
+
}
47
+
48
+
}
+44
src/applications/search/index/PhabricatorIndexEngineExtensionModule.php
+44
src/applications/search/index/PhabricatorIndexEngineExtensionModule.php
···
1
+
<?php
2
+
3
+
final class PhabricatorIndexEngineExtensionModule
4
+
extends PhabricatorConfigModule {
5
+
6
+
public function getModuleKey() {
7
+
return 'indexengine';
8
+
}
9
+
10
+
public function getModuleName() {
11
+
return pht('Engine: Index');
12
+
}
13
+
14
+
public function renderModuleStatus(AphrontRequest $request) {
15
+
$viewer = $request->getViewer();
16
+
17
+
$extensions = PhabricatorIndexEngineExtension::getAllExtensions();
18
+
19
+
$rows = array();
20
+
foreach ($extensions as $extension) {
21
+
$rows[] = array(
22
+
get_class($extension),
23
+
$extension->getExtensionName(),
24
+
);
25
+
}
26
+
27
+
$table = id(new AphrontTableView($rows))
28
+
->setHeaders(
29
+
array(
30
+
pht('Class'),
31
+
pht('Name'),
32
+
))
33
+
->setColumnClasses(
34
+
array(
35
+
null,
36
+
'wide pri',
37
+
));
38
+
39
+
return id(new PHUIObjectBoxView())
40
+
->setHeaderText(pht('IndexEngine Extensions'))
41
+
->setTable($table);
42
+
}
43
+
44
+
}
+1
-14
src/applications/search/index/PhabricatorSearchDocumentIndexer.php
+1
-14
src/applications/search/index/PhabricatorSearchDocumentIndexer.php
···
2
2
3
3
abstract class PhabricatorSearchDocumentIndexer extends Phobject {
4
4
5
-
private $context;
6
-
7
-
protected function setContext($context) {
8
-
$this->context = $context;
9
-
return $this;
10
-
}
11
-
12
-
protected function getContext() {
13
-
return $this->context;
14
-
}
15
-
16
5
abstract public function getIndexableObject();
17
6
abstract protected function buildAbstractDocumentByPHID($phid);
18
7
···
41
30
return $object;
42
31
}
43
32
44
-
public function indexDocumentByPHID($phid, $context) {
45
-
$this->setContext($context);
46
-
33
+
public function indexDocumentByPHID($phid) {
47
34
$document = $this->buildAbstractDocumentByPHID($phid);
48
35
if ($document === null) {
49
36
// This indexer doesn't build a document index, so we're done.
+13
-1
src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php
+13
-1
src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php
···
30
30
'it more difficult to debug search indexing.'),
31
31
),
32
32
array(
33
+
'name' => 'force',
34
+
'short' => 'f',
35
+
'help' => pht(
36
+
'Force a complete rebuild of the entire index instead of an '.
37
+
'incremental update.'),
38
+
),
39
+
array(
33
40
'name' => 'objects',
34
41
'wildcard' => true,
35
42
),
···
41
48
42
49
$is_all = $args->getArg('all');
43
50
$is_type = $args->getArg('type');
51
+
$is_force = $args->getArg('force');
44
52
45
53
$obj_names = $args->getArg('objects');
46
54
···
93
101
$bar = id(new PhutilConsoleProgressBar())
94
102
->setTotal(count($phids));
95
103
104
+
$parameters = array(
105
+
'force' => $is_force,
106
+
);
107
+
96
108
$any_success = false;
97
109
foreach ($phids as $phid) {
98
110
try {
99
-
PhabricatorSearchWorker::queueDocumentForIndexing($phid);
111
+
PhabricatorSearchWorker::queueDocumentForIndexing($phid, $parameters);
100
112
$any_success = true;
101
113
} catch (Exception $ex) {
102
114
phlog($ex);
+24
-6
src/applications/search/worker/PhabricatorSearchWorker.php
+24
-6
src/applications/search/worker/PhabricatorSearchWorker.php
···
2
2
3
3
final class PhabricatorSearchWorker extends PhabricatorWorker {
4
4
5
-
public static function queueDocumentForIndexing($phid, $context = null) {
5
+
public static function queueDocumentForIndexing($phid, $parameters = null) {
6
+
if ($parameters === null) {
7
+
$parameters = array();
8
+
}
9
+
6
10
parent::scheduleTask(
7
11
__CLASS__,
8
12
array(
9
13
'documentPHID' => $phid,
10
-
'context' => $context,
14
+
'parameters' => $parameters,
11
15
),
12
16
array(
13
17
'priority' => parent::PRIORITY_IMPORT,
···
17
21
protected function doWork() {
18
22
$data = $this->getTaskData();
19
23
$object_phid = idx($data, 'documentPHID');
20
-
$context = idx($data, 'context');
21
24
22
-
$engine = new PhabricatorIndexEngine();
25
+
$object = $this->loadObjectForIndexing($object_phid);
26
+
27
+
$engine = id(new PhabricatorIndexEngine())
28
+
->setObject($object);
29
+
30
+
$parameters = idx($data, 'parameters', array());
31
+
$engine->setParameters($parameters);
32
+
33
+
if (!$engine->shouldIndexObject()) {
34
+
return;
35
+
}
23
36
24
37
$key = "index.{$object_phid}";
25
38
$lock = PhabricatorGlobalLock::newLock($key);
···
27
40
$lock->lock(1);
28
41
29
42
try {
30
-
$object = $this->loadObjectForIndexing($object_phid);
43
+
// Reload the object now that we have a lock, to make sure we have the
44
+
// most current version.
45
+
$object = $this->loadObjectForIndexing($object->getPHID());
46
+
47
+
$engine->setObject($object);
31
48
32
-
$engine->indexDocumentByPHID($object->getPHID(), $context);
49
+
$engine->indexObject();
33
50
51
+
$engine->indexDocumentByPHID($object->getPHID());
34
52
} catch (Exception $ex) {
35
53
$lock->unlock();
36
54
+3
-10
src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php
+3
-10
src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php
···
1095
1095
if ($this->supportsSearch()) {
1096
1096
PhabricatorSearchWorker::queueDocumentForIndexing(
1097
1097
$object->getPHID(),
1098
-
$this->getSearchContextParameter($object, $xactions));
1098
+
array(
1099
+
'transactionPHIDs' => mpull($xactions, 'getPHID'),
1100
+
));
1099
1101
}
1100
1102
1101
1103
if ($this->shouldPublishFeedStory($object, $xactions)) {
···
2860
2862
*/
2861
2863
protected function supportsSearch() {
2862
2864
return false;
2863
-
}
2864
-
2865
-
/**
2866
-
* @task search
2867
-
*/
2868
-
protected function getSearchContextParameter(
2869
-
PhabricatorLiskDAO $object,
2870
-
array $xactions) {
2871
-
return null;
2872
2865
}
2873
2866
2874
2867