@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Build a Conpherence thread index
Summary:
Ref T3165. Builds a dedicated index for Conpherence to avoid scale/policy filtering concerns.
- This is pretty one-off but I think it's generally OK.
- There's no UI for it.
- `ConpherenceFulltextQuery` is very low-level. You would need to do another query on the PHIDs it returns to actually show anything to the user.
- The `previousTransactionPHID` is so you can load chat context efficiently. Specifically, if you want to show results like this:
> previous line of context
> **line of chat that matches the query**
> next line of context
...you can read the previous lines out of `previousTransactionPHID` directly, and the next lines by issuing one query with `WHERE previousTransactionPHID IN (...)`.
I'm not 100% sure this is useful, but it seemed like a reasonable thing to provide, since there's no way to query this efficiently otherwise and I figure a lot of chat might make way more sense with a couple of lines of context.
Test Plan:
- Indexed a thread manually (whole thing indexed).
- Indexed a thread by updating it (just the new comment indexed).
- Wrote a hacky test script and got reasonable-looking query results.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: epriestley
Maniphest Tasks: T3165
Differential Revision: https://secure.phabricator.com/D11234
+276
-20
+14
resources/sql/autopatches/20150105.conpsearch.sql
+14
resources/sql/autopatches/20150105.conpsearch.sql
···
1
+
CREATE TABLE {$NAMESPACE}_conpherence.conpherence_index (
2
+
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
3
+
threadPHID VARBINARY(64) NOT NULL,
4
+
transactionPHID VARBINARY(64) NOT NULL,
5
+
previousTransactionPHID VARBINARY(64),
6
+
corpus longtext
7
+
CHARACTER SET {$CHARSET_FULLTEXT}
8
+
COLLATE {$COLLATE_FULLTEXT}
9
+
NOT NULL,
10
+
KEY `key_thread` (threadPHID),
11
+
UNIQUE KEY `key_transaction` (transactionPHID),
12
+
UNIQUE KEY `key_previous` (previousTransactionPHID),
13
+
FULLTEXT KEY `key_corpus` (corpus)
14
+
) ENGINE=MyISAM DEFAULT CHARSET={$CHARSET} COLLATE={$COLLATE_TEXT};
+7
src/__phutil_library_map__.php
+7
src/__phutil_library_map__.php
···
230
230
'ConpherenceDAO' => 'applications/conpherence/storage/ConpherenceDAO.php',
231
231
'ConpherenceEditor' => 'applications/conpherence/editor/ConpherenceEditor.php',
232
232
'ConpherenceFileWidgetView' => 'applications/conpherence/view/ConpherenceFileWidgetView.php',
233
+
'ConpherenceFulltextQuery' => 'applications/conpherence/query/ConpherenceFulltextQuery.php',
233
234
'ConpherenceHovercardEventListener' => 'applications/conpherence/events/ConpherenceHovercardEventListener.php',
235
+
'ConpherenceIndex' => 'applications/conpherence/storage/ConpherenceIndex.php',
234
236
'ConpherenceLayoutView' => 'applications/conpherence/view/ConpherenceLayoutView.php',
235
237
'ConpherenceListController' => 'applications/conpherence/controller/ConpherenceListController.php',
236
238
'ConpherenceMenuItemView' => 'applications/conpherence/view/ConpherenceMenuItemView.php',
···
247
249
'ConpherenceSchemaSpec' => 'applications/conpherence/storage/ConpherenceSchemaSpec.php',
248
250
'ConpherenceSettings' => 'applications/conpherence/constants/ConpherenceSettings.php',
249
251
'ConpherenceThread' => 'applications/conpherence/storage/ConpherenceThread.php',
252
+
'ConpherenceThreadIndexer' => 'applications/conpherence/search/ConpherenceThreadIndexer.php',
250
253
'ConpherenceThreadListView' => 'applications/conpherence/view/ConpherenceThreadListView.php',
251
254
'ConpherenceThreadMailReceiver' => 'applications/conpherence/mail/ConpherenceThreadMailReceiver.php',
252
255
'ConpherenceThreadQuery' => 'applications/conpherence/query/ConpherenceThreadQuery.php',
···
3290
3293
'ConpherenceDAO' => 'PhabricatorLiskDAO',
3291
3294
'ConpherenceEditor' => 'PhabricatorApplicationTransactionEditor',
3292
3295
'ConpherenceFileWidgetView' => 'ConpherenceWidgetView',
3296
+
'ConpherenceFulltextQuery' => 'PhabricatorOffsetPagedQuery',
3293
3297
'ConpherenceHovercardEventListener' => 'PhabricatorEventListener',
3298
+
'ConpherenceIndex' => 'ConpherenceDAO',
3294
3299
'ConpherenceLayoutView' => 'AphrontView',
3295
3300
'ConpherenceListController' => 'ConpherenceController',
3296
3301
'ConpherenceMenuItemView' => 'AphrontTagView',
···
3310
3315
'ConpherenceDAO',
3311
3316
'PhabricatorPolicyInterface',
3312
3317
),
3318
+
'ConpherenceThreadIndexer' => 'PhabricatorSearchDocumentIndexer',
3313
3319
'ConpherenceThreadListView' => 'AphrontView',
3314
3320
'ConpherenceThreadMailReceiver' => 'PhabricatorObjectMailReceiver',
3315
3321
'ConpherenceThreadQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
···
5603
5609
'PhabricatorSearchDeleteController' => 'PhabricatorSearchBaseController',
5604
5610
'PhabricatorSearchDocument' => 'PhabricatorSearchDAO',
5605
5611
'PhabricatorSearchDocumentField' => 'PhabricatorSearchDAO',
5612
+
'PhabricatorSearchDocumentIndexer' => 'Phobject',
5606
5613
'PhabricatorSearchDocumentQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
5607
5614
'PhabricatorSearchDocumentRelationship' => 'PhabricatorSearchDAO',
5608
5615
'PhabricatorSearchEditController' => 'PhabricatorSearchBaseController',
+17
-1
src/applications/conpherence/editor/ConpherenceEditor.php
+17
-1
src/applications/conpherence/editor/ConpherenceEditor.php
···
459
459
}
460
460
461
461
protected function supportsSearch() {
462
-
return false;
462
+
return true;
463
+
}
464
+
465
+
protected function getSearchContextParameter(
466
+
PhabricatorLiskDAO $object,
467
+
array $xactions) {
468
+
469
+
$comment_phids = array();
470
+
foreach ($xactions as $xaction) {
471
+
if ($xaction->hasComment()) {
472
+
$comment_phids[] = $xaction->getPHID();
473
+
}
474
+
}
475
+
476
+
return array(
477
+
'commentPHIDs' => $comment_phids,
478
+
);
463
479
}
464
480
465
481
}
+68
src/applications/conpherence/query/ConpherenceFulltextQuery.php
+68
src/applications/conpherence/query/ConpherenceFulltextQuery.php
···
1
+
<?php
2
+
3
+
final class ConpherenceFulltextQuery
4
+
extends PhabricatorOffsetPagedQuery {
5
+
6
+
private $threadPHIDs;
7
+
private $fulltext;
8
+
9
+
public function withThreadPHIDs(array $phids) {
10
+
$this->threadPHIDs = $phids;
11
+
return $this;
12
+
}
13
+
14
+
public function withFulltext($fulltext) {
15
+
$this->fulltext = $fulltext;
16
+
return $this;
17
+
}
18
+
19
+
public function execute() {
20
+
$table = new ConpherenceIndex();
21
+
$conn_r = $table->establishConnection('r');
22
+
23
+
$rows = queryfx_all(
24
+
$conn_r,
25
+
'SELECT threadPHID, transactionPHID, previousTransactionPHID
26
+
FROM %T i %Q %Q %Q',
27
+
$table->getTableName(),
28
+
$this->buildWhereClause($conn_r),
29
+
$this->buildOrderByClause($conn_r),
30
+
$this->buildLimitClause($conn_r));
31
+
32
+
return $rows;
33
+
}
34
+
35
+
private function buildWhereClause($conn_r) {
36
+
$where = array();
37
+
38
+
if ($this->threadPHIDs !== null) {
39
+
$where[] = qsprintf(
40
+
$conn_r,
41
+
'i.threadPHID IN (%Ls)',
42
+
$this->threadPHIDs);
43
+
}
44
+
45
+
if (strlen($this->fulltext)) {
46
+
$where[] = qsprintf(
47
+
$conn_r,
48
+
'MATCH(i.corpus) AGAINST (%s IN BOOLEAN MODE)',
49
+
$this->fulltext);
50
+
}
51
+
52
+
return $this->formatWhereClause($where);
53
+
}
54
+
55
+
private function buildOrderByClause(AphrontDatabaseConnection $conn_r) {
56
+
if (strlen($this->fulltext)) {
57
+
return qsprintf(
58
+
$conn_r,
59
+
'ORDER BY MATCH(i.corpus) AGAINST (%s IN BOOLEAN MODE) DESC',
60
+
$this->fulltext);
61
+
} else {
62
+
return qsprintf(
63
+
$conn_r,
64
+
'ORDER BY id DESC');
65
+
}
66
+
}
67
+
68
+
}
+89
src/applications/conpherence/search/ConpherenceThreadIndexer.php
+89
src/applications/conpherence/search/ConpherenceThreadIndexer.php
···
1
+
<?php
2
+
3
+
final class ConpherenceThreadIndexer
4
+
extends PhabricatorSearchDocumentIndexer {
5
+
6
+
public function getIndexableObject() {
7
+
return new ConpherenceThread();
8
+
}
9
+
10
+
protected function loadDocumentByPHID($phid) {
11
+
$object = id(new ConpherenceThreadQuery())
12
+
->setViewer($this->getViewer())
13
+
->withPHIDs(array($phid))
14
+
->executeOne();
15
+
16
+
if (!$object) {
17
+
throw new Exception(pht('No thread "%s" exists!', $phid));
18
+
}
19
+
20
+
return $object;
21
+
}
22
+
23
+
protected function buildAbstractDocumentByPHID($phid) {
24
+
$thread = $this->loadDocumentByPHID($phid);
25
+
26
+
// NOTE: We're explicitly not building a document here, only rebuilding
27
+
// the Conpherence search index.
28
+
29
+
$context = nonempty($this->getContext(), array());
30
+
$comment_phids = idx($context, 'commentPHIDs');
31
+
32
+
if (is_array($comment_phids) && !$comment_phids) {
33
+
// If this property is set, but empty, the transaction did not
34
+
// include any chat text. For example, a user might have left the
35
+
// conversation.
36
+
return null;
37
+
}
38
+
39
+
$query = id(new ConpherenceTransactionQuery())
40
+
->setViewer($this->getViewer())
41
+
->withObjectPHIDs(array($thread->getPHID()))
42
+
->withTransactionTypes(array(PhabricatorTransactions::TYPE_COMMENT))
43
+
->needComments(true);
44
+
45
+
if ($comment_phids !== null) {
46
+
$query->withPHIDs($comment_phids);
47
+
}
48
+
49
+
$xactions = $query->execute();
50
+
51
+
foreach ($xactions as $xaction) {
52
+
$this->indexComment($thread, $xaction);
53
+
}
54
+
55
+
return null;
56
+
}
57
+
58
+
private function indexComment(
59
+
ConpherenceThread $thread,
60
+
ConpherenceTransaction $xaction) {
61
+
62
+
$previous = id(new ConpherenceTransactionQuery())
63
+
->setViewer($this->getViewer())
64
+
->withObjectPHIDs(array($thread->getPHID()))
65
+
->withTransactionTypes(array(PhabricatorTransactions::TYPE_COMMENT))
66
+
->setAfterID($xaction->getID())
67
+
->setLimit(1)
68
+
->executeOne();
69
+
70
+
$index = id(new ConpherenceIndex())
71
+
->setThreadPHID($thread->getPHID())
72
+
->setTransactionPHID($xaction->getPHID())
73
+
->setPreviousTransactionPHID($previous ? $previous->getPHID() : null)
74
+
->setCorpus($xaction->getComment()->getContent());
75
+
76
+
queryfx(
77
+
$index->establishConnection('w'),
78
+
'INSERT INTO %T
79
+
(threadPHID, transactionPHID, previousTransactionPHID, corpus)
80
+
VALUES (%s, %s, %ns, %s)
81
+
ON DUPLICATE KEY UPDATE corpus = VALUES(corpus)',
82
+
$index->getTableName(),
83
+
$index->getThreadPHID(),
84
+
$index->getTransactionPHID(),
85
+
$index->getPreviousTransactionPHID(),
86
+
$index->getCorpus());
87
+
}
88
+
89
+
}
+38
src/applications/conpherence/storage/ConpherenceIndex.php
+38
src/applications/conpherence/storage/ConpherenceIndex.php
···
1
+
<?php
2
+
3
+
final class ConpherenceIndex
4
+
extends ConpherenceDAO {
5
+
6
+
protected $threadPHID;
7
+
protected $transactionPHID;
8
+
protected $previousTransactionPHID;
9
+
protected $corpus;
10
+
11
+
public function getConfiguration() {
12
+
return array(
13
+
self::CONFIG_TIMESTAMPS => false,
14
+
self::CONFIG_COLUMN_SCHEMA => array(
15
+
'previousTransactionPHID' => 'phid?',
16
+
'corpus' => 'fulltext',
17
+
),
18
+
self::CONFIG_KEY_SCHEMA => array(
19
+
'key_thread' => array(
20
+
'columns' => array('threadPHID'),
21
+
),
22
+
'key_transaction' => array(
23
+
'columns' => array('transactionPHID'),
24
+
'unique' => true,
25
+
),
26
+
'key_previous' => array(
27
+
'columns' => array('previousTransactionPHID'),
28
+
'unique' => true,
29
+
),
30
+
'key_corpus' => array(
31
+
'columns' => array('corpus'),
32
+
'type' => 'FULLTEXT',
33
+
),
34
+
),
35
+
) + parent::getConfiguration();
36
+
}
37
+
38
+
}
+19
-2
src/applications/search/index/PhabricatorSearchDocumentIndexer.php
+19
-2
src/applications/search/index/PhabricatorSearchDocumentIndexer.php
···
1
1
<?php
2
2
3
-
abstract class PhabricatorSearchDocumentIndexer {
3
+
abstract class PhabricatorSearchDocumentIndexer extends Phobject {
4
+
5
+
private $context;
6
+
7
+
protected function setContext($context) {
8
+
$this->context = $context;
9
+
return $this;
10
+
}
11
+
12
+
protected function getContext() {
13
+
return $this->context;
14
+
}
4
15
5
16
abstract public function getIndexableObject();
6
17
abstract protected function buildAbstractDocumentByPHID($phid);
···
30
41
return $object;
31
42
}
32
43
33
-
public function indexDocumentByPHID($phid) {
44
+
public function indexDocumentByPHID($phid, $context) {
34
45
try {
46
+
$this->setContext($context);
47
+
35
48
$document = $this->buildAbstractDocumentByPHID($phid);
49
+
if ($document === null) {
50
+
// This indexer doesn't build a document index, so we're done.
51
+
return $this;
52
+
}
36
53
37
54
$object = $this->loadDocumentByPHID($phid);
38
55
+4
-3
src/applications/search/index/PhabricatorSearchIndexer.php
+4
-3
src/applications/search/index/PhabricatorSearchIndexer.php
···
2
2
3
3
final class PhabricatorSearchIndexer {
4
4
5
-
public function queueDocumentForIndexing($phid) {
5
+
public function queueDocumentForIndexing($phid, $context = null) {
6
6
PhabricatorWorker::scheduleTask(
7
7
'PhabricatorSearchWorker',
8
8
array(
9
9
'documentPHID' => $phid,
10
+
'context' => $context,
10
11
),
11
12
array(
12
13
'priority' => PhabricatorWorker::PRIORITY_IMPORT,
13
14
));
14
15
}
15
16
16
-
public function indexDocumentByPHID($phid) {
17
+
public function indexDocumentByPHID($phid, $context) {
17
18
$indexers = id(new PhutilSymbolLoader())
18
19
->setAncestorClass('PhabricatorSearchDocumentIndexer')
19
20
->loadObjects();
20
21
21
22
foreach ($indexers as $indexer) {
22
23
if ($indexer->shouldIndexDocumentByPHID($phid)) {
23
-
$indexer->indexDocumentByPHID($phid);
24
+
$indexer->indexDocumentByPHID($phid, $context);
24
25
break;
25
26
}
26
27
}
+3
-10
src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php
+3
-10
src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php
···
114
114
}
115
115
116
116
private function loadPHIDsByTypes($type) {
117
-
$indexer_symbols = id(new PhutilSymbolLoader())
118
-
->setAncestorClass('PhabricatorSearchDocumentIndexer')
119
-
->setConcreteOnly(true)
120
-
->setType('class')
121
-
->selectAndLoadSymbols();
122
-
123
-
$indexers = array();
124
-
foreach ($indexer_symbols as $symbol) {
125
-
$indexers[] = newv($symbol['name'], array());
126
-
}
117
+
$indexers = id(new PhutilSymbolLoader())
118
+
->setAncestorClass('PhabricatorSearchObjectIndexer')
119
+
->loadObjects();
127
120
128
121
$phids = array();
129
122
foreach ($indexers as $indexer) {
+3
-1
src/applications/search/worker/PhabricatorSearchWorker.php
+3
-1
src/applications/search/worker/PhabricatorSearchWorker.php
···
4
4
5
5
public function doWork() {
6
6
$data = $this->getTaskData();
7
+
7
8
$phid = idx($data, 'documentPHID');
9
+
$context = idx($data, 'context');
8
10
9
11
id(new PhabricatorSearchIndexer())
10
-
->indexDocumentByPHID($phid);
12
+
->indexDocumentByPHID($phid, $context);
11
13
}
12
14
13
15
}
+12
-1
src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php
+12
-1
src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php
···
800
800
801
801
if ($this->supportsSearch()) {
802
802
id(new PhabricatorSearchIndexer())
803
-
->queueDocumentForIndexing($object->getPHID());
803
+
->queueDocumentForIndexing(
804
+
$object->getPHID(),
805
+
$this->getSearchContextParameter($object, $xactions));
804
806
}
805
807
806
808
if ($this->shouldPublishFeedStory($object, $xactions)) {
···
2353
2355
*/
2354
2356
protected function supportsSearch() {
2355
2357
return false;
2358
+
}
2359
+
2360
+
/**
2361
+
* @task search
2362
+
*/
2363
+
protected function getSearchContextParameter(
2364
+
PhabricatorLiskDAO $object,
2365
+
array $xactions) {
2366
+
return null;
2356
2367
}
2357
2368
2358
2369
+2
-2
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php
+2
-2
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php
···
82
82
'{$NAMESPACE}',
83
83
$dump);
84
84
85
-
// NOTE: This is a hack. We can not use `binary` for this column, because
86
-
// it is part of a fulltext index.
85
+
// NOTE: This is a hack. We can not use `binary` for these columns, because
86
+
// they are a part of a fulltext index.
87
87
$old = $dump;
88
88
$dump = preg_replace(
89
89
'/`corpus` longtext CHARACTER SET .* COLLATE .*,/mi',