@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Improve Search architecture
Summary:
The search indexing API has several problems right now:
- Always runs in-process.
- It would be nice to push this into the task queue for performance. However, the API currently passses an object all the way through (and some indexers depend on preloaded object attributes), so it can't be dumped into the task queue at any stage since we can't serialize it.
- Being able to use the task queue will also make rebuilding indexes faster.
- Instead, make the API phid-oriented.
- No uniform indexing API.
- Each "Editor" currently calls SomeCustomIndexer::indexThing(). This won't work with AbstractTransactions. The API is also just weird.
- Instead, provide a uniform API.
- No uniform CLI.
- We have `scripts/search/reindex_everything.php`, but it doesn't actually index everything. Each new document type needs to be separately added to it, leading to stuff like D3839. Third-party applications can't provide indexers.
- Instead, let indexers expose documents for indexing.
- Not application-oriented.
- All the indexers live in search/ right now, which isn't the right organization in an application-orietned view of the world.
- Instead, move indexers to applications and load them with SymbolLoader.
Test Plan:
- `bin/search index`
- Indexed one revision, one task.
- Indexed `--type TASK`, `--type DREV`, etc., for all types.
- Indexed `--all`.
- Added the word "saboteur" to a revision, task, wiki page, and question and then searched for it.
- Creating users is a pain; searched for a user after indexing.
- Creating commits is a pain; searched for a commit after indexing.
- Mocks aren't currently loadable in the result view, so their indexing is moot.
Reviewers: btrahan, vrana
Reviewed By: btrahan
CC: 20after4, aran
Maniphest Tasks: T1991, T2104
Differential Revision: https://secure.phabricator.com/D4261
+386
-161
+1
bin/search
+1
bin/search
···
1
+
../scripts/search/manage_search.php
+2
-33
scripts/search/index_one_commit.php
+2
-33
scripts/search/index_one_commit.php
···
1
1
#!/usr/bin/env php
2
2
<?php
3
3
4
-
$root = dirname(dirname(dirname(__FILE__)));
5
-
require_once $root.'/scripts/__init_script__.php';
6
-
7
-
if (empty($argv[1])) {
8
-
echo "usage: index_one_commit.php <commit_name>\n";
9
-
die(1);
10
-
}
11
-
12
-
$commit = isset($argv[1]) ? $argv[1] : null;
13
-
if (!$commit) {
14
-
throw new Exception("Provide a commit to index!");
15
-
}
16
-
$matches = null;
17
-
if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $commit, $matches)) {
18
-
throw new Exception("Can't parse commit identifier!");
19
-
}
20
-
$repo = id(new PhabricatorRepository())->loadOneWhere(
21
-
'callsign = %s',
22
-
$matches[1]);
23
-
if (!$repo) {
24
-
throw new Exception("Unknown repository!");
25
-
}
26
-
27
-
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
28
-
'repositoryID = %d AND commitIdentifier = %s',
29
-
$repo->getID(),
30
-
$matches[2]);
31
-
if (!$commit) {
32
-
throw new Exception('Unknown commit.');
33
-
}
34
-
35
-
PhabricatorSearchCommitIndexer::indexCommit($commit);
36
-
echo "Done.\n";
4
+
echo "Use 'bin/search index rXnnnnnn' instead of this script.\n";
5
+
exit(1);
+22
scripts/search/manage_search.php
+22
scripts/search/manage_search.php
···
1
+
#!/usr/bin/env php
2
+
<?php
3
+
4
+
$root = dirname(dirname(dirname(__FILE__)));
5
+
require_once $root.'/scripts/__init_script__.php';
6
+
7
+
$args = new PhutilArgumentParser($argv);
8
+
$args->setTagline('manage search');
9
+
$args->setSynopsis(<<<EOSYNOPSIS
10
+
**search** __command__ [__options__]
11
+
Manage Phabricator search index.
12
+
13
+
EOSYNOPSIS
14
+
);
15
+
$args->parseStandardArguments();
16
+
17
+
$workflows = array(
18
+
new PhabricatorSearchManagementIndexWorkflow(),
19
+
new PhutilHelpArgumentWorkflow(),
20
+
);
21
+
22
+
$args->parseWorkflows($workflows);
+2
-12
scripts/search/reindex_all_users.php
+2
-12
scripts/search/reindex_all_users.php
···
1
1
#!/usr/bin/env php
2
2
<?php
3
3
4
-
$root = dirname(dirname(dirname(__FILE__)));
5
-
require_once $root.'/scripts/__init_script__.php';
6
-
7
-
$users = id(new PhabricatorUser())->loadAll();
8
-
echo "Indexing ".count($users)." users";
9
-
foreach ($users as $user) {
10
-
PhabricatorSearchUserIndexer::indexUser($user);
11
-
echo '.';
12
-
}
13
-
echo "\n";
14
-
echo "Done.\n";
15
-
4
+
echo "Use 'bin/search index --type USER' instead of this script.\n";
5
+
die(1);
+2
-31
scripts/search/reindex_everything.php
+2
-31
scripts/search/reindex_everything.php
···
1
1
#!/usr/bin/env php
2
2
<?php
3
3
4
-
$root = dirname(dirname(dirname(__FILE__)));
5
-
require_once $root.'/scripts/__init_script__.php';
6
-
7
-
// TODO: Get rid of this script eventually, once this stuff is better-formalized
8
-
// in Timeline consumers.
9
-
10
-
echo "Reindexing revisions...\n";
11
-
$revs = new LiskMigrationIterator(new DifferentialRevision());
12
-
foreach ($revs as $rev) {
13
-
PhabricatorSearchDifferentialIndexer::indexRevision($rev);
14
-
echo '.';
15
-
}
16
-
echo "\n";
17
-
18
-
echo "Reindexing commits...\n";
19
-
$commits = new LiskMigrationIterator(new PhabricatorRepositoryCommit());
20
-
foreach ($commits as $commit) {
21
-
PhabricatorSearchCommitIndexer::indexCommit($commit);
22
-
echo '.';
23
-
}
24
-
echo "\n";
25
-
26
-
echo "Reindexing tasks...\n";
27
-
$tasks = new LiskMigrationIterator(new ManiphestTask());
28
-
foreach ($tasks as $task) {
29
-
PhabricatorSearchManiphestIndexer::indexTask($task);
30
-
echo '.';
31
-
}
32
-
echo "\n";
33
-
34
-
include dirname(__FILE__).'/reindex_all_users.php';
4
+
echo "Use 'bin/search index --all' instead of this script.\n";
5
+
die(1);
+20
-15
src/__phutil_library_map__.php
+20
-15
src/__phutil_library_map__.php
···
310
310
'DifferentialRevisionStatusFieldSpecification' => 'applications/differential/field/specification/DifferentialRevisionStatusFieldSpecification.php',
311
311
'DifferentialRevisionUpdateHistoryView' => 'applications/differential/view/DifferentialRevisionUpdateHistoryView.php',
312
312
'DifferentialRevisionViewController' => 'applications/differential/controller/DifferentialRevisionViewController.php',
313
+
'DifferentialSearchIndexer' => 'applications/differential/search/DifferentialSearchIndexer.php',
313
314
'DifferentialSubscribeController' => 'applications/differential/controller/DifferentialSubscribeController.php',
314
315
'DifferentialSummaryFieldSpecification' => 'applications/differential/field/specification/DifferentialSummaryFieldSpecification.php',
315
316
'DifferentialTasksAttacher' => 'applications/differential/DifferentialTasksAttacher.php',
···
533
534
'ManiphestSavedQueryDeleteController' => 'applications/maniphest/controller/ManiphestSavedQueryDeleteController.php',
534
535
'ManiphestSavedQueryEditController' => 'applications/maniphest/controller/ManiphestSavedQueryEditController.php',
535
536
'ManiphestSavedQueryListController' => 'applications/maniphest/controller/ManiphestSavedQueryListController.php',
537
+
'ManiphestSearchIndexer' => 'applications/maniphest/search/ManiphestSearchIndexer.php',
536
538
'ManiphestSubpriorityController' => 'applications/maniphest/controller/ManiphestSubpriorityController.php',
537
539
'ManiphestTask' => 'applications/maniphest/storage/ManiphestTask.php',
538
540
'ManiphestTaskAuxiliaryStorage' => 'applications/maniphest/storage/ManiphestTaskAuxiliaryStorage.php',
···
1052
1054
'PhabricatorRepositoryCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php',
1053
1055
'PhabricatorRepositoryCommitOwnersWorker' => 'applications/repository/worker/PhabricatorRepositoryCommitOwnersWorker.php',
1054
1056
'PhabricatorRepositoryCommitParserWorker' => 'applications/repository/worker/PhabricatorRepositoryCommitParserWorker.php',
1057
+
'PhabricatorRepositoryCommitSearchIndexer' => 'applications/repository/search/PhabricatorRepositoryCommitSearchIndexer.php',
1055
1058
'PhabricatorRepositoryController' => 'applications/repository/controller/PhabricatorRepositoryController.php',
1056
1059
'PhabricatorRepositoryCreateController' => 'applications/repository/controller/PhabricatorRepositoryCreateController.php',
1057
1060
'PhabricatorRepositoryDAO' => 'applications/repository/storage/PhabricatorRepositoryDAO.php',
···
1084
1087
'PhabricatorSearchAbstractDocument' => 'applications/search/index/PhabricatorSearchAbstractDocument.php',
1085
1088
'PhabricatorSearchAttachController' => 'applications/search/controller/PhabricatorSearchAttachController.php',
1086
1089
'PhabricatorSearchBaseController' => 'applications/search/controller/PhabricatorSearchBaseController.php',
1087
-
'PhabricatorSearchCommitIndexer' => 'applications/search/index/indexer/PhabricatorSearchCommitIndexer.php',
1088
1090
'PhabricatorSearchController' => 'applications/search/controller/PhabricatorSearchController.php',
1089
1091
'PhabricatorSearchDAO' => 'applications/search/storage/PhabricatorSearchDAO.php',
1090
-
'PhabricatorSearchDifferentialIndexer' => 'applications/search/index/indexer/PhabricatorSearchDifferentialIndexer.php',
1091
1092
'PhabricatorSearchDocument' => 'applications/search/storage/document/PhabricatorSearchDocument.php',
1092
1093
'PhabricatorSearchDocumentField' => 'applications/search/storage/document/PhabricatorSearchDocumentField.php',
1093
-
'PhabricatorSearchDocumentIndexer' => 'applications/search/index/indexer/PhabricatorSearchDocumentIndexer.php',
1094
+
'PhabricatorSearchDocumentIndexer' => 'applications/search/index/PhabricatorSearchDocumentIndexer.php',
1094
1095
'PhabricatorSearchDocumentRelationship' => 'applications/search/storage/document/PhabricatorSearchDocumentRelationship.php',
1095
1096
'PhabricatorSearchEngine' => 'applications/search/engine/PhabricatorSearchEngine.php',
1096
1097
'PhabricatorSearchEngineElastic' => 'applications/search/engine/PhabricatorSearchEngineElastic.php',
1097
1098
'PhabricatorSearchEngineMySQL' => 'applications/search/engine/PhabricatorSearchEngineMySQL.php',
1098
1099
'PhabricatorSearchEngineSelector' => 'applications/search/selector/PhabricatorSearchEngineSelector.php',
1099
1100
'PhabricatorSearchField' => 'applications/search/constants/PhabricatorSearchField.php',
1100
-
'PhabricatorSearchManiphestIndexer' => 'applications/search/index/indexer/PhabricatorSearchManiphestIndexer.php',
1101
-
'PhabricatorSearchPhrictionIndexer' => 'applications/search/index/indexer/PhabricatorSearchPhrictionIndexer.php',
1102
-
'PhabricatorSearchPonderIndexer' => 'applications/ponder/search/PhabricatorSearchPonderIndexer.php',
1101
+
'PhabricatorSearchIndexer' => 'applications/search/index/PhabricatorSearchIndexer.php',
1102
+
'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php',
1103
+
'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php',
1103
1104
'PhabricatorSearchQuery' => 'applications/search/storage/PhabricatorSearchQuery.php',
1104
1105
'PhabricatorSearchRelationship' => 'applications/search/constants/PhabricatorSearchRelationship.php',
1105
1106
'PhabricatorSearchResultView' => 'applications/search/view/PhabricatorSearchResultView.php',
1106
1107
'PhabricatorSearchScope' => 'applications/search/constants/PhabricatorSearchScope.php',
1107
1108
'PhabricatorSearchSelectController' => 'applications/search/controller/PhabricatorSearchSelectController.php',
1108
-
'PhabricatorSearchUserIndexer' => 'applications/search/index/indexer/PhabricatorSearchUserIndexer.php',
1109
1109
'PhabricatorSettingsAdjustController' => 'applications/settings/controller/PhabricatorSettingsAdjustController.php',
1110
1110
'PhabricatorSettingsMainController' => 'applications/settings/controller/PhabricatorSettingsMainController.php',
1111
1111
'PhabricatorSettingsPanel' => 'applications/settings/panel/PhabricatorSettingsPanel.php',
···
1191
1191
'PhabricatorUserPreferences' => 'applications/settings/storage/PhabricatorUserPreferences.php',
1192
1192
'PhabricatorUserProfile' => 'applications/people/storage/PhabricatorUserProfile.php',
1193
1193
'PhabricatorUserSSHKey' => 'applications/settings/storage/PhabricatorUserSSHKey.php',
1194
+
'PhabricatorUserSearchIndexer' => 'applications/people/search/PhabricatorUserSearchIndexer.php',
1194
1195
'PhabricatorUserStatus' => 'applications/people/storage/PhabricatorUserStatus.php',
1195
1196
'PhabricatorUserStatusInvalidEpochException' => 'applications/people/exception/PhabricatorUserStatusInvalidEpochException.php',
1196
1197
'PhabricatorUserStatusOverlapException' => 'applications/people/exception/PhabricatorUserStatusOverlapException.php',
···
1256
1257
'PholioController' => 'applications/pholio/controller/PholioController.php',
1257
1258
'PholioDAO' => 'applications/pholio/storage/PholioDAO.php',
1258
1259
'PholioImage' => 'applications/pholio/storage/PholioImage.php',
1259
-
'PholioIndexer' => 'applications/pholio/indexer/PholioIndexer.php',
1260
1260
'PholioMock' => 'applications/pholio/storage/PholioMock.php',
1261
1261
'PholioMockCommentController' => 'applications/pholio/controller/PholioMockCommentController.php',
1262
1262
'PholioMockEditController' => 'applications/pholio/controller/PholioMockEditController.php',
···
1265
1265
'PholioMockQuery' => 'applications/pholio/query/PholioMockQuery.php',
1266
1266
'PholioMockViewController' => 'applications/pholio/controller/PholioMockViewController.php',
1267
1267
'PholioReplyHandler' => 'applications/pholio/mail/PholioReplyHandler.php',
1268
+
'PholioSearchIndexer' => 'applications/pholio/indexer/PholioSearchIndexer.php',
1268
1269
'PholioTransaction' => 'applications/pholio/storage/PholioTransaction.php',
1269
1270
'PholioTransactionComment' => 'applications/pholio/storage/PholioTransactionComment.php',
1270
1271
'PholioTransactionQuery' => 'applications/pholio/query/PholioTransactionQuery.php',
···
1291
1292
'PhrictionHistoryController' => 'applications/phriction/controller/PhrictionHistoryController.php',
1292
1293
'PhrictionListController' => 'applications/phriction/controller/PhrictionListController.php',
1293
1294
'PhrictionNewController' => 'applications/phriction/controller/PhrictionNewController.php',
1295
+
'PhrictionSearchIndexer' => 'applications/phriction/search/PhrictionSearchIndexer.php',
1294
1296
'PonderAddAnswerView' => 'applications/ponder/view/PonderAddAnswerView.php',
1295
1297
'PonderAddCommentView' => 'applications/ponder/view/PonderAddCommentView.php',
1296
1298
'PonderAnswer' => 'applications/ponder/storage/PonderAnswer.php',
···
1324
1326
'PonderQuestionViewController' => 'applications/ponder/controller/PonderQuestionViewController.php',
1325
1327
'PonderReplyHandler' => 'applications/ponder/PonderReplyHandler.php',
1326
1328
'PonderRuleQuestion' => 'infrastructure/markup/rule/PonderRuleQuestion.php',
1329
+
'PonderSearchIndexer' => 'applications/ponder/search/PonderSearchIndexer.php',
1327
1330
'PonderUserProfileView' => 'applications/ponder/view/PonderUserProfileView.php',
1328
1331
'PonderVotableInterface' => 'applications/ponder/storage/PonderVotableInterface.php',
1329
1332
'PonderVotableView' => 'applications/ponder/view/PonderVotableView.php',
···
1631
1634
'DifferentialRevisionStatusFieldSpecification' => 'DifferentialFieldSpecification',
1632
1635
'DifferentialRevisionUpdateHistoryView' => 'AphrontView',
1633
1636
'DifferentialRevisionViewController' => 'DifferentialController',
1637
+
'DifferentialSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
1634
1638
'DifferentialSubscribeController' => 'DifferentialController',
1635
1639
'DifferentialSummaryFieldSpecification' => 'DifferentialFreeformFieldSpecification',
1636
1640
'DifferentialTestPlanFieldSpecification' => 'DifferentialFieldSpecification',
···
1815
1819
'ManiphestSavedQueryDeleteController' => 'ManiphestController',
1816
1820
'ManiphestSavedQueryEditController' => 'ManiphestController',
1817
1821
'ManiphestSavedQueryListController' => 'ManiphestController',
1822
+
'ManiphestSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
1818
1823
'ManiphestSubpriorityController' => 'ManiphestController',
1819
1824
'ManiphestTask' =>
1820
1825
array(
···
2324
2329
'PhabricatorRepositoryCommitMessageParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
2325
2330
'PhabricatorRepositoryCommitOwnersWorker' => 'PhabricatorRepositoryCommitParserWorker',
2326
2331
'PhabricatorRepositoryCommitParserWorker' => 'PhabricatorWorker',
2332
+
'PhabricatorRepositoryCommitSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
2327
2333
'PhabricatorRepositoryController' => 'PhabricatorController',
2328
2334
'PhabricatorRepositoryCreateController' => 'PhabricatorRepositoryController',
2329
2335
'PhabricatorRepositoryDAO' => 'PhabricatorLiskDAO',
···
2351
2357
'PhabricatorSSHWorkflow' => 'PhutilArgumentWorkflow',
2352
2358
'PhabricatorSearchAttachController' => 'PhabricatorSearchBaseController',
2353
2359
'PhabricatorSearchBaseController' => 'PhabricatorController',
2354
-
'PhabricatorSearchCommitIndexer' => 'PhabricatorSearchDocumentIndexer',
2355
2360
'PhabricatorSearchController' => 'PhabricatorSearchBaseController',
2356
2361
'PhabricatorSearchDAO' => 'PhabricatorLiskDAO',
2357
-
'PhabricatorSearchDifferentialIndexer' => 'PhabricatorSearchDocumentIndexer',
2358
2362
'PhabricatorSearchDocument' => 'PhabricatorSearchDAO',
2359
2363
'PhabricatorSearchDocumentField' => 'PhabricatorSearchDAO',
2360
2364
'PhabricatorSearchDocumentRelationship' => 'PhabricatorSearchDAO',
2361
2365
'PhabricatorSearchEngineElastic' => 'PhabricatorSearchEngine',
2362
2366
'PhabricatorSearchEngineMySQL' => 'PhabricatorSearchEngine',
2363
-
'PhabricatorSearchManiphestIndexer' => 'PhabricatorSearchDocumentIndexer',
2364
-
'PhabricatorSearchPhrictionIndexer' => 'PhabricatorSearchDocumentIndexer',
2365
-
'PhabricatorSearchPonderIndexer' => 'PhabricatorSearchDocumentIndexer',
2367
+
'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow',
2368
+
'PhabricatorSearchManagementWorkflow' => 'PhutilArgumentWorkflow',
2366
2369
'PhabricatorSearchQuery' => 'PhabricatorSearchDAO',
2367
2370
'PhabricatorSearchResultView' => 'AphrontView',
2368
2371
'PhabricatorSearchSelectController' => 'PhabricatorSearchBaseController',
2369
-
'PhabricatorSearchUserIndexer' => 'PhabricatorSearchDocumentIndexer',
2370
2372
'PhabricatorSettingsAdjustController' => 'PhabricatorController',
2371
2373
'PhabricatorSettingsMainController' => 'PhabricatorController',
2372
2374
'PhabricatorSettingsPanelAccount' => 'PhabricatorSettingsPanel',
···
2444
2446
'PhabricatorUserPreferences' => 'PhabricatorUserDAO',
2445
2447
'PhabricatorUserProfile' => 'PhabricatorUserDAO',
2446
2448
'PhabricatorUserSSHKey' => 'PhabricatorUserDAO',
2449
+
'PhabricatorUserSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
2447
2450
'PhabricatorUserStatus' => 'PhabricatorUserDAO',
2448
2451
'PhabricatorUserStatusInvalidEpochException' => 'Exception',
2449
2452
'PhabricatorUserStatusOverlapException' => 'Exception',
···
2520
2523
0 => 'PholioDAO',
2521
2524
1 => 'PhabricatorMarkupInterface',
2522
2525
),
2523
-
'PholioIndexer' => 'PhabricatorSearchDocumentIndexer',
2524
2526
'PholioMock' =>
2525
2527
array(
2526
2528
0 => 'PholioDAO',
···
2535
2537
'PholioMockQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
2536
2538
'PholioMockViewController' => 'PholioController',
2537
2539
'PholioReplyHandler' => 'PhabricatorMailReplyHandler',
2540
+
'PholioSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
2538
2541
'PholioTransaction' => 'PhabricatorApplicationTransaction',
2539
2542
'PholioTransactionComment' => 'PhabricatorApplicationTransactionComment',
2540
2543
'PholioTransactionQuery' => 'PhabricatorApplicationTransactionQuery',
···
2564
2567
'PhrictionHistoryController' => 'PhrictionController',
2565
2568
'PhrictionListController' => 'PhrictionController',
2566
2569
'PhrictionNewController' => 'PhrictionController',
2570
+
'PhrictionSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
2567
2571
'PonderAddAnswerView' => 'AphrontView',
2568
2572
'PonderAddCommentView' => 'AphrontView',
2569
2573
'PonderAnswer' =>
···
2611
2615
'PonderQuestionViewController' => 'PonderController',
2612
2616
'PonderReplyHandler' => 'PhabricatorMailReplyHandler',
2613
2617
'PonderRuleQuestion' => 'PhabricatorRemarkupRuleObjectName',
2618
+
'PonderSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
2614
2619
'PonderUserProfileView' => 'AphrontView',
2615
2620
'PonderVotableView' => 'AphrontView',
2616
2621
'PonderVoteEditor' => 'PhabricatorEditor',
+4
-1
src/applications/audit/editor/PhabricatorAuditCommentEditor.php
+4
-1
src/applications/audit/editor/PhabricatorAuditCommentEditor.php
···
289
289
290
290
$feed_phids = array_diff($requests_phids, $feed_dont_publish_phids);
291
291
$this->publishFeedStory($comment, $feed_phids);
292
-
PhabricatorSearchCommitIndexer::indexCommit($commit);
292
+
293
+
id(new PhabricatorSearchIndexer())
294
+
->indexDocumentByPHID($commit->getPHID());
295
+
293
296
$this->sendMail($comment, $other_comments, $inline_comments, $requests);
294
297
}
295
298
+2
-2
src/applications/differential/editor/DifferentialCommentEditor.php
+2
-2
src/applications/differential/editor/DifferentialCommentEditor.php
···
598
598
->setMailRecipientPHIDs($mailed_phids)
599
599
->publish();
600
600
601
-
// TODO: Move to workers
602
-
PhabricatorSearchDifferentialIndexer::indexRevision($revision);
601
+
id(new PhabricatorSearchIndexer())
602
+
->indexDocumentByPHID($revision->getPHID());
603
603
604
604
return $comment;
605
605
}
+2
-2
src/applications/differential/editor/DifferentialRevisionEditor.php
+2
-2
src/applications/differential/editor/DifferentialRevisionEditor.php
···
489
489
->setMailRecipientPHIDs($mailed_phids)
490
490
->publish();
491
491
492
-
// TODO: Move this into a worker task thing.
493
-
PhabricatorSearchDifferentialIndexer::indexRevision($revision);
492
+
id(new PhabricatorSearchIndexer())
493
+
->indexDocumentByPHID($revision->getPHID());
494
494
}
495
495
496
496
public static function addCCAndUpdateRevision(
+2
-1
src/applications/diffusion/controller/DiffusionCommitEditController.php
+2
-1
src/applications/diffusion/controller/DiffusionCommitEditController.php
···
44
44
}
45
45
$editor->save();
46
46
47
-
PhabricatorSearchCommitIndexer::indexCommit($commit);
47
+
id(new PhabricatorSearchIndexer())
48
+
->indexDocumentByPHID($commit->getPHID());
48
49
49
50
return id(new AphrontRedirectResponse())
50
51
->setURI('/r'.$callsign.$commit->getCommitIdentifier());
+2
-3
src/applications/maniphest/editor/ManiphestTransactionEditor.php
+2
-3
src/applications/maniphest/editor/ManiphestTransactionEditor.php
···
199
199
$transactions,
200
200
$mail->buildRecipientList());
201
201
202
-
// TODO: Do this offline via workers
203
-
PhabricatorSearchManiphestIndexer::indexTask($task);
204
-
202
+
id(new PhabricatorSearchIndexer())
203
+
->indexDocumentByPHID($task->getPHID());
205
204
}
206
205
207
206
protected function getSubjectPrefix() {
+3
-1
src/applications/people/storage/PhabricatorUser.php
+3
-1
src/applications/people/storage/PhabricatorUser.php
+9
-3
src/applications/pholio/indexer/PholioIndexer.php
src/applications/pholio/indexer/PholioSearchIndexer.php
+9
-3
src/applications/pholio/indexer/PholioIndexer.php
src/applications/pholio/indexer/PholioSearchIndexer.php
···
3
3
/**
4
4
* @group pholio
5
5
*/
6
-
final class PholioIndexer extends PhabricatorSearchDocumentIndexer {
6
+
final class PholioSearchIndexer extends PhabricatorSearchDocumentIndexer {
7
7
8
-
public static function indexMock(PholioMock $mock) {
8
+
public function getIndexableObject() {
9
+
return new PholioMock();
10
+
}
11
+
12
+
protected function buildAbstractDocumentByPHID($phid) {
13
+
$mock = $this->loadDocumentByPHID($phid);
14
+
9
15
$doc = new PhabricatorSearchAbstractDocument();
10
16
$doc->setPHID($mock->getPHID());
11
17
$doc->setDocumentType(phid_get_type($mock->getPHID()));
···
23
29
PhabricatorPHIDConstants::PHID_TYPE_USER,
24
30
$mock->getDateCreated());
25
31
26
-
self::reindexAbstractDocument($doc);
32
+
return $doc;
27
33
}
28
34
}
+3
-1
src/applications/phriction/editor/PhrictionDocumentEditor.php
+3
-1
src/applications/phriction/editor/PhrictionDocumentEditor.php
···
172
172
$document->save();
173
173
174
174
$document->attachContent($new_content);
175
-
PhabricatorSearchPhrictionIndexer::indexDocument($document);
175
+
176
+
id(new PhabricatorSearchIndexer())
177
+
->indexDocumentByPHID($document->getPHID());
176
178
177
179
$project_phid = null;
178
180
$slug = $document->getSlug();
+2
-1
src/applications/ponder/editor/PonderAnswerEditor.php
+2
-1
src/applications/ponder/editor/PonderAnswerEditor.php
···
47
47
$trans->saveTransaction();
48
48
49
49
$question->attachRelated();
50
-
PhabricatorSearchPonderIndexer::indexQuestion($question);
50
+
id(new PhabricatorSearchIndexer())
51
+
->indexDocumentByPHID($question->getPHID());
51
52
52
53
// subscribe author and @mentions
53
54
$subeditor = id(new PhabricatorSubscriptionsEditor())
+2
-2
src/applications/ponder/editor/PonderCommentEditor.php
+2
-2
src/applications/ponder/editor/PonderCommentEditor.php
···
39
39
$target = $this->targetPHID;
40
40
$comment->save();
41
41
42
-
$question->attachRelated();
43
-
PhabricatorSearchPonderIndexer::indexQuestion($question);
42
+
id(new PhabricatorSearchIndexer())
43
+
->indexDocumentByPHID($question->getPHID());
44
44
45
45
// subscribe author and @mentions
46
46
$subeditor = id(new PhabricatorSubscriptionsEditor())
+2
-2
src/applications/ponder/editor/PonderQuestionEditor.php
+2
-2
src/applications/ponder/editor/PonderQuestionEditor.php
···
24
24
$question = $this->question;
25
25
$question->save();
26
26
27
-
// search index
28
27
$question->attachRelated();
29
-
PhabricatorSearchPonderIndexer::indexQuestion($question);
28
+
id(new PhabricatorSearchIndexer())
29
+
->indexDocumentByPHID($question->getPHID());
30
30
31
31
// subscribe author and @mentions
32
32
$subeditor = id(new PhabricatorSubscriptionsEditor())
+10
-4
src/applications/ponder/search/PhabricatorSearchPonderIndexer.php
src/applications/ponder/search/PonderSearchIndexer.php
+10
-4
src/applications/ponder/search/PhabricatorSearchPonderIndexer.php
src/applications/ponder/search/PonderSearchIndexer.php
···
1
1
<?php
2
2
3
-
final class PhabricatorSearchPonderIndexer
3
+
final class PonderSearchIndexer
4
4
extends PhabricatorSearchDocumentIndexer {
5
5
6
-
public static function indexQuestion(PonderQuestion $question) {
7
-
// note: we assume someone's already called attachrelated on $question
6
+
public function getIndexableObject() {
7
+
return new PonderQuestion();
8
+
}
9
+
10
+
protected function buildAbstractDocumentByPHID($phid) {
11
+
$question = $this->loadDocumentByPHID($phid);
12
+
13
+
$question->attachRelated();
8
14
9
15
$doc = new PhabricatorSearchAbstractDocument();
10
16
$doc->setPHID($question->getPHID());
···
61
67
$question->getDateModified()); // Bogus timestamp.
62
68
}
63
69
64
-
self::reindexAbstractDocument($doc);
70
+
return $doc;
65
71
}
66
72
}
+3
-1
src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php
+3
-1
src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php
···
57
57
58
58
protected function finishParse() {
59
59
$commit = $this->commit;
60
-
PhabricatorSearchCommitIndexer::indexCommit($commit);
60
+
61
+
id(new PhabricatorSearchIndexer())
62
+
->indexDocumentByPHID($commit->getPHID());
61
63
62
64
PhabricatorOwnersPackagePathValidator::updateOwnersPackagePaths($commit);
63
65
if ($this->shouldQueueFollowupTasks()) {
+54
src/applications/search/index/PhabricatorSearchDocumentIndexer.php
+54
src/applications/search/index/PhabricatorSearchDocumentIndexer.php
···
1
+
<?php
2
+
3
+
/**
4
+
* @group search
5
+
*/
6
+
abstract class PhabricatorSearchDocumentIndexer {
7
+
8
+
abstract public function getIndexableObject();
9
+
abstract protected function buildAbstractDocumentByPHID($phid);
10
+
11
+
public function shouldIndexDocumentByPHID($phid) {
12
+
$object = $this->getIndexableObject();
13
+
return (phid_get_type($phid) == phid_get_type($object->generatePHID()));
14
+
}
15
+
16
+
public function getIndexIterator() {
17
+
$object = $this->getIndexableObject();
18
+
return new LiskMigrationIterator($object);
19
+
}
20
+
21
+
protected function loadDocumentByPHID($phid) {
22
+
$object = $this->getIndexableObject();
23
+
$document = $object->loadOneWhere('phid = %s', $phid);
24
+
if (!$document) {
25
+
throw new Exception("Unable to load document by phid '{$phid}'!");
26
+
}
27
+
return $document;
28
+
}
29
+
30
+
public function indexDocumentByPHID($phid) {
31
+
try {
32
+
$document = $this->buildAbstractDocumentByPHID($phid);
33
+
34
+
$engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
35
+
try {
36
+
$engine->reindexAbstractDocument($document);
37
+
} catch (Exception $ex) {
38
+
$phid = $document->getPHID();
39
+
$class = get_class($engine);
40
+
41
+
phlog("Unable to index document {$phid} by engine {$class}.");
42
+
phlog($ex);
43
+
}
44
+
45
+
} catch (Exception $ex) {
46
+
$class = get_class($this);
47
+
phlog("Unable to build document {$phid} by indexer {$class}.");
48
+
phlog($ex);
49
+
}
50
+
51
+
return $this;
52
+
}
53
+
54
+
}
+30
src/applications/search/index/PhabricatorSearchIndexer.php
+30
src/applications/search/index/PhabricatorSearchIndexer.php
···
1
+
<?php
2
+
3
+
/**
4
+
* @group search
5
+
*/
6
+
final class PhabricatorSearchIndexer {
7
+
8
+
public function indexDocumentByPHID($phid) {
9
+
$doc_indexer_symbols = id(new PhutilSymbolLoader())
10
+
->setAncestorClass('PhabricatorSearchDocumentIndexer')
11
+
->setConcreteOnly(true)
12
+
->setType('class')
13
+
->selectAndLoadSymbols();
14
+
15
+
$indexers = array();
16
+
foreach ($doc_indexer_symbols as $symbol) {
17
+
$indexers[] = newv($symbol['name'], array());
18
+
}
19
+
20
+
foreach ($indexers as $indexer) {
21
+
if ($indexer->shouldIndexDocumentByPHID($phid)) {
22
+
$indexer->indexDocumentByPHID($phid);
23
+
break;
24
+
}
25
+
}
26
+
27
+
return $this;
28
+
}
29
+
30
+
}
+10
-7
src/applications/search/index/indexer/PhabricatorSearchCommitIndexer.php
src/applications/repository/search/PhabricatorRepositoryCommitSearchIndexer.php
+10
-7
src/applications/search/index/indexer/PhabricatorSearchCommitIndexer.php
src/applications/repository/search/PhabricatorRepositoryCommitSearchIndexer.php
···
1
1
<?php
2
2
3
-
/**
4
-
* @group search
5
-
*/
6
-
final class PhabricatorSearchCommitIndexer
3
+
final class PhabricatorRepositoryCommitSearchIndexer
7
4
extends PhabricatorSearchDocumentIndexer {
8
5
9
-
public static function indexCommit(PhabricatorRepositoryCommit $commit) {
6
+
public function getIndexableObject() {
7
+
return new PhabricatorRepositoryCommit();
8
+
}
9
+
10
+
protected function buildAbstractDocumentByPHID($phid) {
11
+
$commit = $this->loadDocumentByPHID($phid);
12
+
10
13
$commit_data = id(new PhabricatorRepositoryCommitData())->loadOneWhere(
11
14
'commitID = %d',
12
15
$commit->getID());
···
19
22
$commit->getRepositoryID());
20
23
21
24
if (!$repository) {
22
-
return;
25
+
throw new Exception("No such repository!");
23
26
}
24
27
25
28
$title = 'r'.$repository->getCallsign().$commit->getCommitIdentifier().
···
75
78
}
76
79
}
77
80
78
-
self::reindexAbstractDocument($doc);
81
+
return $doc;
79
82
}
80
83
}
81
84
+10
-4
src/applications/search/index/indexer/PhabricatorSearchDifferentialIndexer.php
src/applications/differential/search/DifferentialSearchIndexer.php
+10
-4
src/applications/search/index/indexer/PhabricatorSearchDifferentialIndexer.php
src/applications/differential/search/DifferentialSearchIndexer.php
···
1
1
<?php
2
2
3
3
/**
4
-
* @group search
4
+
* @group differential
5
5
*/
6
-
final class PhabricatorSearchDifferentialIndexer
6
+
final class DifferentialSearchIndexer
7
7
extends PhabricatorSearchDocumentIndexer {
8
8
9
-
public static function indexRevision(DifferentialRevision $rev) {
9
+
public function getIndexableObject() {
10
+
return new DifferentialRevision();
11
+
}
12
+
13
+
protected function buildAbstractDocumentByPHID($phid) {
14
+
$rev = $this->loadDocumentByPHID($phid);
15
+
10
16
$doc = new PhabricatorSearchAbstractDocument();
11
17
$doc->setPHID($rev->getPHID());
12
18
$doc->setDocumentType(PhabricatorPHIDConstants::PHID_TYPE_DREV);
···
108
114
$rev->getDateModified()); // Bogus timestamp.
109
115
}
110
116
111
-
self::reindexAbstractDocument($doc);
117
+
return $doc;
112
118
}
113
119
}
-21
src/applications/search/index/indexer/PhabricatorSearchDocumentIndexer.php
-21
src/applications/search/index/indexer/PhabricatorSearchDocumentIndexer.php
···
1
-
<?php
2
-
3
-
/**
4
-
* @group search
5
-
*/
6
-
abstract class PhabricatorSearchDocumentIndexer {
7
-
8
-
// TODO: Make this whole class tree concrete?
9
-
final protected static function reindexAbstractDocument(
10
-
PhabricatorSearchAbstractDocument $document) {
11
-
$engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
12
-
try {
13
-
$engine->reindexAbstractDocument($document);
14
-
} catch (Exception $ex) {
15
-
$phid = $document->getPHID();
16
-
$class = get_class($engine);
17
-
phlog("Unable to index document {$phid} by engine {$class}.");
18
-
}
19
-
}
20
-
21
-
}
+10
-4
src/applications/search/index/indexer/PhabricatorSearchManiphestIndexer.php
src/applications/maniphest/search/ManiphestSearchIndexer.php
+10
-4
src/applications/search/index/indexer/PhabricatorSearchManiphestIndexer.php
src/applications/maniphest/search/ManiphestSearchIndexer.php
···
1
1
<?php
2
2
3
3
/**
4
-
* @group search
4
+
* @group maniphest
5
5
*/
6
-
final class PhabricatorSearchManiphestIndexer
6
+
final class ManiphestSearchIndexer
7
7
extends PhabricatorSearchDocumentIndexer {
8
8
9
-
public static function indexTask(ManiphestTask $task) {
9
+
public function getIndexableObject() {
10
+
return new ManiphestTask();
11
+
}
12
+
13
+
protected function buildAbstractDocumentByPHID($phid) {
14
+
$task = $this->loadDocumentByPHID($phid);
15
+
10
16
$doc = new PhabricatorSearchAbstractDocument();
11
17
$doc->setPHID($task->getPHID());
12
18
$doc->setDocumentType(PhabricatorPHIDConstants::PHID_TYPE_TASK);
···
114
120
$time);
115
121
}
116
122
117
-
self::reindexAbstractDocument($doc);
123
+
return $doc;
118
124
}
119
125
}
+13
-4
src/applications/search/index/indexer/PhabricatorSearchPhrictionIndexer.php
src/applications/phriction/search/PhrictionSearchIndexer.php
+13
-4
src/applications/search/index/indexer/PhabricatorSearchPhrictionIndexer.php
src/applications/phriction/search/PhrictionSearchIndexer.php
···
1
1
<?php
2
2
3
3
/**
4
-
* @group search
4
+
* @group phriction
5
5
*/
6
-
final class PhabricatorSearchPhrictionIndexer
6
+
final class PhrictionSearchIndexer
7
7
extends PhabricatorSearchDocumentIndexer {
8
8
9
-
public static function indexDocument(PhrictionDocument $document) {
9
+
public function getIndexableObject() {
10
+
return new PhrictionDocument();
11
+
}
12
+
13
+
protected function buildAbstractDocumentByPHID($phid) {
14
+
$document = $this->loadDocumentByPHID($phid);
15
+
16
+
$content = id(new PhrictionContent())->load($document->getContentID());
17
+
$document->attachContent($content);
18
+
10
19
$content = $document->getContent();
11
20
12
21
$doc = new PhabricatorSearchAbstractDocument();
···
28
37
PhabricatorPHIDConstants::PHID_TYPE_USER,
29
38
$content->getDateCreated());
30
39
31
-
self::reindexAbstractDocument($doc);
40
+
return $doc;
32
41
}
33
42
}
+9
-6
src/applications/search/index/indexer/PhabricatorSearchUserIndexer.php
src/applications/people/search/PhabricatorUserSearchIndexer.php
+9
-6
src/applications/search/index/indexer/PhabricatorSearchUserIndexer.php
src/applications/people/search/PhabricatorUserSearchIndexer.php
···
1
1
<?php
2
2
3
-
/**
4
-
* @group search
5
-
*/
6
-
final class PhabricatorSearchUserIndexer
3
+
final class PhabricatorUserSearchIndexer
7
4
extends PhabricatorSearchDocumentIndexer {
8
5
9
-
public static function indexUser(PhabricatorUser $user) {
6
+
public function getIndexableObject() {
7
+
return new PhabricatorUser();
8
+
}
9
+
10
+
protected function buildAbstractDocumentByPHID($phid) {
11
+
$user = $this->loadDocumentByPHID($phid);
12
+
10
13
$doc = new PhabricatorSearchAbstractDocument();
11
14
$doc->setPHID($user->getPHID());
12
15
$doc->setDocumentType(PhabricatorPHIDConstants::PHID_TYPE_USER);
···
25
28
time());
26
29
}
27
30
28
-
self::reindexAbstractDocument($doc);
31
+
return $doc;
29
32
}
30
33
}
+137
src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php
+137
src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php
···
1
+
<?php
2
+
3
+
/**
4
+
* @group search
5
+
*/
6
+
final class PhabricatorSearchManagementIndexWorkflow
7
+
extends PhabricatorSearchManagementWorkflow {
8
+
9
+
protected function didConstruct() {
10
+
$this
11
+
->setName('index')
12
+
->setSynopsis('Build or rebuild search indexes.')
13
+
->setExamples(
14
+
"**index** D123\n".
15
+
"**index** --type DREV\n".
16
+
"**index** --all")
17
+
->setArguments(
18
+
array(
19
+
array(
20
+
'name' => 'all',
21
+
'help' => 'Reindex all documents.',
22
+
),
23
+
array(
24
+
'name' => 'type',
25
+
'param' => 'TYPE',
26
+
'help' => 'PHID type to reindex, like "TASK" or "DREV".',
27
+
),
28
+
array(
29
+
'name' => 'background',
30
+
'help' => 'Instead of indexing in this process, queue tasks for '.
31
+
'the daemons. This is better if you are indexing a lot '.
32
+
'of stuff, but less helpful for debugging.',
33
+
),
34
+
array(
35
+
'name' => 'foreground',
36
+
'help' => 'Index in this process, even if there are many objects '.
37
+
'to index. This is helpful for debugging.',
38
+
),
39
+
array(
40
+
'name' => 'objects',
41
+
'wildcard' => true,
42
+
),
43
+
)
44
+
);
45
+
}
46
+
47
+
public function execute(PhutilArgumentParser $args) {
48
+
$console = PhutilConsole::getConsole();
49
+
50
+
$is_all = $args->getArg('all');
51
+
$is_type = $args->getArg('type');
52
+
53
+
$obj_names = $args->getArg('objects');
54
+
55
+
56
+
if ($obj_names && ($is_all || $is_type)) {
57
+
throw new PhutilArgumentUsageException(
58
+
"You can not name objects to index alongside the '--all' or '--type' ".
59
+
"flags.");
60
+
} else if (!$obj_names && !($is_all || $is_type)) {
61
+
throw new PhutilArgumentUsageException(
62
+
"Provide one of '--all', '--type' or a list of object names.");
63
+
}
64
+
65
+
if ($obj_names) {
66
+
$phids = $this->loadPHIDsByNames($obj_names);
67
+
} else {
68
+
$phids = $this->loadPHIDsByTypes($is_type);
69
+
}
70
+
71
+
if (!$phids) {
72
+
throw new PhutilArgumentUsageException(
73
+
"Nothing to index!");
74
+
}
75
+
76
+
$groups = phid_group_by_type($phids);
77
+
foreach ($groups as $group_type => $group) {
78
+
$console->writeOut(
79
+
pht(
80
+
"Indexing %d object(s) of type %s.",
81
+
count($group),
82
+
$group_type)."\n");
83
+
}
84
+
85
+
$indexer = new PhabricatorSearchIndexer();
86
+
foreach ($phids as $phid) {
87
+
$indexer->indexDocumentByPHID($phid);
88
+
$console->writeOut(pht("Indexing '%s'...\n", $phid));
89
+
}
90
+
91
+
$console->writeOut("Done.\n");
92
+
}
93
+
94
+
private function loadPHIDsByNames(array $names) {
95
+
$phids = array();
96
+
foreach ($names as $name) {
97
+
$phid = PhabricatorPHID::fromObjectName($name);
98
+
if (!$phid) {
99
+
throw new PhutilArgumentUsageException(
100
+
"'{$name}' is not the name of a known object.");
101
+
}
102
+
$phids[] = $phid;
103
+
}
104
+
return $phids;
105
+
}
106
+
107
+
private function loadPHIDsByTypes($type) {
108
+
$indexer_symbols = id(new PhutilSymbolLoader())
109
+
->setAncestorClass('PhabricatorSearchDocumentIndexer')
110
+
->setConcreteOnly(true)
111
+
->setType('class')
112
+
->selectAndLoadSymbols();
113
+
114
+
$indexers = array();
115
+
foreach ($indexer_symbols as $symbol) {
116
+
$indexers[] = newv($symbol['name'], array());
117
+
}
118
+
119
+
$phids = array();
120
+
foreach ($indexers as $indexer) {
121
+
$indexer_phid = $indexer->getIndexableObject()->generatePHID();
122
+
$indexer_type = phid_get_type($indexer_phid);
123
+
124
+
if ($type && ($indexer_type != $type)) {
125
+
continue;
126
+
}
127
+
128
+
$iterator = $indexer->getIndexIterator();
129
+
foreach ($iterator as $object) {
130
+
$phids[] = $object->getPHID();
131
+
}
132
+
}
133
+
134
+
return $phids;
135
+
}
136
+
137
+
}
+13
src/applications/search/management/PhabricatorSearchManagementWorkflow.php
+13
src/applications/search/management/PhabricatorSearchManagementWorkflow.php