@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Begin improving modularity of IndexEngine, add locks
Summary:
Ref T9890. Ref T9979. Several adjacent goals:
- The `SearchEngine` vs `ApplicationSearchEngine` thing is really confusing. There are also a bunch of confusing class names and class relationships within the fulltext indexing. I want to rename these classes to be more standard (`IndexEngine`, `IndexEngineExtension`, etc). Rename `SearchIndexer` to `IndexEngine`. A future change will rename `SearchEngine`.
- Add the index locks described in T9890.
- Structure things a little more normally so future diffs can do the "EngineExtension" thing more cleanly.
Test Plan:
Indexing:
- Renamed a task to have a unique word in the title.
- Ran `bin/search index Txxx`.
- Searched for unique word.
- Found task.
Locking:
- Added a `sleep(10)` after the `lock()` call.
- Ran `bin/search index Txxx` in two windows.
- Saw first one lock, sleep 10 seconds, index.
- Saw second one give up temporarily after failing to grab the lock.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T9890, T9979
Differential Revision: https://secure.phabricator.com/D14834
+83
-55
+2
-2
src/__phutil_library_map__.php
+2
-2
src/__phutil_library_map__.php
···
2374
2374
'PhabricatorImageMacroRemarkupRule' => 'applications/macro/markup/PhabricatorImageMacroRemarkupRule.php',
2375
2375
'PhabricatorImageTransformer' => 'applications/files/PhabricatorImageTransformer.php',
2376
2376
'PhabricatorImagemagickSetupCheck' => 'applications/config/check/PhabricatorImagemagickSetupCheck.php',
2377
+
'PhabricatorIndexEngine' => 'applications/search/index/PhabricatorIndexEngine.php',
2377
2378
'PhabricatorInfrastructureTestCase' => '__tests__/PhabricatorInfrastructureTestCase.php',
2378
2379
'PhabricatorInlineCommentController' => 'infrastructure/diff/PhabricatorInlineCommentController.php',
2379
2380
'PhabricatorInlineCommentInterface' => 'infrastructure/diff/interface/PhabricatorInlineCommentInterface.php',
···
3032
3033
'PhabricatorSearchEngineTestCase' => 'applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php',
3033
3034
'PhabricatorSearchField' => 'applications/search/field/PhabricatorSearchField.php',
3034
3035
'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php',
3035
-
'PhabricatorSearchIndexer' => 'applications/search/index/PhabricatorSearchIndexer.php',
3036
3036
'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php',
3037
3037
'PhabricatorSearchManagementInitWorkflow' => 'applications/search/management/PhabricatorSearchManagementInitWorkflow.php',
3038
3038
'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php',
···
6596
6596
'PhabricatorImageMacroRemarkupRule' => 'PhutilRemarkupRule',
6597
6597
'PhabricatorImageTransformer' => 'Phobject',
6598
6598
'PhabricatorImagemagickSetupCheck' => 'PhabricatorSetupCheck',
6599
+
'PhabricatorIndexEngine' => 'Phobject',
6599
6600
'PhabricatorInfrastructureTestCase' => 'PhabricatorTestCase',
6600
6601
'PhabricatorInlineCommentController' => 'PhabricatorController',
6601
6602
'PhabricatorInlineCommentInterface' => 'PhabricatorMarkupInterface',
···
7378
7379
'PhabricatorSearchEngineTestCase' => 'PhabricatorTestCase',
7379
7380
'PhabricatorSearchField' => 'Phobject',
7380
7381
'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController',
7381
-
'PhabricatorSearchIndexer' => 'Phobject',
7382
7382
'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow',
7383
7383
'PhabricatorSearchManagementInitWorkflow' => 'PhabricatorSearchManagementWorkflow',
7384
7384
'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow',
+2
-4
src/applications/diviner/publisher/DivinerLivePublisher.php
+2
-4
src/applications/diviner/publisher/DivinerLivePublisher.php
···
40
40
$conn_w->saveTransaction();
41
41
$this->book = $book;
42
42
43
-
id(new PhabricatorSearchIndexer())
44
-
->queueDocumentForIndexing($book->getPHID());
43
+
PhabricatorSearchWorker::queueDocumentForIndexing($book->getPHID());
45
44
}
46
45
47
46
return $this->book;
···
147
146
148
147
$symbol->save();
149
148
150
-
id(new PhabricatorSearchIndexer())
151
-
->queueDocumentForIndexing($symbol->getPHID());
149
+
PhabricatorSearchWorker::queueDocumentForIndexing($symbol->getPHID());
152
150
153
151
// TODO: We probably need a finer-grained sense of what "documentable"
154
152
// atoms are. Neither files nor methods are currently considered
+1
-2
src/applications/people/storage/PhabricatorUser.php
+1
-2
src/applications/people/storage/PhabricatorUser.php
+1
-2
src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php
+1
-2
src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php
···
93
93
$commit->writeImportStatusFlag(
94
94
PhabricatorRepositoryCommit::IMPORTED_CHANGE);
95
95
96
-
id(new PhabricatorSearchIndexer())
97
-
->queueDocumentForIndexing($commit->getPHID());
96
+
PhabricatorSearchWorker::queueDocumentForIndexing($commit->getPHID());
98
97
99
98
if ($this->shouldQueueFollowupTasks()) {
100
99
$this->queueTask(
+20
src/applications/search/index/PhabricatorIndexEngine.php
+20
src/applications/search/index/PhabricatorIndexEngine.php
···
1
+
<?php
2
+
3
+
final class PhabricatorIndexEngine extends Phobject {
4
+
5
+
public function indexDocumentByPHID($phid, $context) {
6
+
$indexers = id(new PhutilClassMapQuery())
7
+
->setAncestorClass('PhabricatorSearchDocumentIndexer')
8
+
->execute();
9
+
10
+
foreach ($indexers as $indexer) {
11
+
if ($indexer->shouldIndexDocumentByPHID($phid)) {
12
+
$indexer->indexDocumentByPHID($phid, $context);
13
+
break;
14
+
}
15
+
}
16
+
17
+
return $this;
18
+
}
19
+
20
+
}
-32
src/applications/search/index/PhabricatorSearchIndexer.php
-32
src/applications/search/index/PhabricatorSearchIndexer.php
···
1
-
<?php
2
-
3
-
final class PhabricatorSearchIndexer extends Phobject {
4
-
5
-
public function queueDocumentForIndexing($phid, $context = null) {
6
-
PhabricatorWorker::scheduleTask(
7
-
'PhabricatorSearchWorker',
8
-
array(
9
-
'documentPHID' => $phid,
10
-
'context' => $context,
11
-
),
12
-
array(
13
-
'priority' => PhabricatorWorker::PRIORITY_IMPORT,
14
-
));
15
-
}
16
-
17
-
public function indexDocumentByPHID($phid, $context) {
18
-
$indexers = id(new PhutilClassMapQuery())
19
-
->setAncestorClass('PhabricatorSearchDocumentIndexer')
20
-
->execute();
21
-
22
-
foreach ($indexers as $indexer) {
23
-
if ($indexer->shouldIndexDocumentByPHID($phid)) {
24
-
$indexer->indexDocumentByPHID($phid, $context);
25
-
break;
26
-
}
27
-
}
28
-
29
-
return $this;
30
-
}
31
-
32
-
}
+1
-2
src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php
+1
-2
src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php
···
94
94
->setTotal(count($phids));
95
95
96
96
$any_success = false;
97
-
$indexer = new PhabricatorSearchIndexer();
98
97
foreach ($phids as $phid) {
99
98
try {
100
-
$indexer->queueDocumentForIndexing($phid);
99
+
PhabricatorSearchWorker::queueDocumentForIndexing($phid);
101
100
$any_success = true;
102
101
} catch (Exception $ex) {
103
102
phlog($ex);
+53
-7
src/applications/search/worker/PhabricatorSearchWorker.php
+53
-7
src/applications/search/worker/PhabricatorSearchWorker.php
···
2
2
3
3
final class PhabricatorSearchWorker extends PhabricatorWorker {
4
4
5
+
public static function queueDocumentForIndexing($phid, $context = null) {
6
+
parent::scheduleTask(
7
+
__CLASS__,
8
+
array(
9
+
'documentPHID' => $phid,
10
+
'context' => $context,
11
+
),
12
+
array(
13
+
'priority' => parent::PRIORITY_IMPORT,
14
+
));
15
+
}
16
+
5
17
protected function doWork() {
6
18
$data = $this->getTaskData();
7
-
8
-
$phid = idx($data, 'documentPHID');
19
+
$object_phid = idx($data, 'documentPHID');
9
20
$context = idx($data, 'context');
10
21
22
+
$engine = new PhabricatorIndexEngine();
23
+
24
+
$key = "index.{$object_phid}";
25
+
$lock = PhabricatorGlobalLock::newLock($key);
26
+
27
+
$lock->lock(1);
28
+
11
29
try {
12
-
id(new PhabricatorSearchIndexer())
13
-
->indexDocumentByPHID($phid, $context);
30
+
$object = $this->loadObjectForIndexing($object_phid);
31
+
32
+
$engine->indexDocumentByPHID($object->getPHID(), $context);
33
+
14
34
} catch (Exception $ex) {
35
+
$lock->unlock();
36
+
37
+
if (!($ex instanceof PhabricatorWorkerPermanentFailureException)) {
38
+
$ex = new PhabricatorWorkerPermanentFailureException(
39
+
pht(
40
+
'Failed to update search index for document "%s": %s',
41
+
$object_phid,
42
+
$ex->getMessage()));
43
+
}
44
+
45
+
throw $ex;
46
+
}
47
+
48
+
$lock->unlock();
49
+
}
50
+
51
+
private function loadObjectForIndexing($phid) {
52
+
$viewer = PhabricatorUser::getOmnipotentUser();
53
+
54
+
$object = id(new PhabricatorObjectQuery())
55
+
->setViewer($viewer)
56
+
->withPHIDs(array($phid))
57
+
->executeOne();
58
+
59
+
if (!$object) {
15
60
throw new PhabricatorWorkerPermanentFailureException(
16
61
pht(
17
-
'Failed to update search index for document "%s": %s',
18
-
$phid,
19
-
$ex->getMessage()));
62
+
'Unable to load object "%s" to rebuild indexes.',
63
+
$phid));
20
64
}
65
+
66
+
return $object;
21
67
}
22
68
23
69
}
+3
-4
src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php
+3
-4
src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php
···
1093
1093
}
1094
1094
1095
1095
if ($this->supportsSearch()) {
1096
-
id(new PhabricatorSearchIndexer())
1097
-
->queueDocumentForIndexing(
1098
-
$object->getPHID(),
1099
-
$this->getSearchContextParameter($object, $xactions));
1096
+
PhabricatorSearchWorker::queueDocumentForIndexing(
1097
+
$object->getPHID(),
1098
+
$this->getSearchContextParameter($object, $xactions));
1100
1099
}
1101
1100
1102
1101
if ($this->shouldPublishFeedStory($object, $xactions)) {