@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add a garbage collector for common ngrams
Summary:
Ref T13000. After an ngram is marked as "common", we can delete it from the storage table.
Currently, the only way to get ngrams marked as "common" is to manually run `bin/search ngrams`, so this has no impact on normal installs.
Test Plan: Ran `bin/garbage collect`, saw it start chewing through my local Maniphest ngrams table and removing common ngrams.
Reviewers: amckinley
Reviewed By: amckinley
Maniphest Tasks: T13000
Differential Revision: https://secure.phabricator.com/D18687
+61
-1
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
3941
3941
'PhabricatorSearchEngineAttachment' => 'applications/search/engineextension/PhabricatorSearchEngineAttachment.php',
3942
3942
'PhabricatorSearchEngineExtension' => 'applications/search/engineextension/PhabricatorSearchEngineExtension.php',
3943
3943
'PhabricatorSearchEngineExtensionModule' => 'applications/search/engineextension/PhabricatorSearchEngineExtensionModule.php',
3944
+
'PhabricatorSearchFerretNgramGarbageCollector' => 'applications/search/garbagecollector/PhabricatorSearchFerretNgramGarbageCollector.php',
3944
3945
'PhabricatorSearchField' => 'applications/search/field/PhabricatorSearchField.php',
3945
3946
'PhabricatorSearchHost' => 'infrastructure/cluster/search/PhabricatorSearchHost.php',
3946
3947
'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php',
···
9522
9523
'PhabricatorSearchEngineAttachment' => 'Phobject',
9523
9524
'PhabricatorSearchEngineExtension' => 'Phobject',
9524
9525
'PhabricatorSearchEngineExtensionModule' => 'PhabricatorConfigModule',
9526
+
'PhabricatorSearchFerretNgramGarbageCollector' => 'PhabricatorGarbageCollector',
9525
9527
'PhabricatorSearchField' => 'Phobject',
9526
9528
'PhabricatorSearchHost' => 'Phobject',
9527
9529
'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController',
+55
src/applications/search/garbagecollector/PhabricatorSearchFerretNgramGarbageCollector.php
+55
src/applications/search/garbagecollector/PhabricatorSearchFerretNgramGarbageCollector.php
···
1
+
<?php
2
+
3
+
final class PhabricatorSearchFerretNgramGarbageCollector
4
+
extends PhabricatorGarbageCollector {
5
+
6
+
const COLLECTORCONST = 'search.ferret.ngram';
7
+
8
+
public function getCollectorName() {
9
+
return pht('Ferret Engine Ngrams');
10
+
}
11
+
12
+
public function hasAutomaticPolicy() {
13
+
return true;
14
+
}
15
+
16
+
protected function collectGarbage() {
17
+
$all_objects = id(new PhutilClassMapQuery())
18
+
->setAncestorClass('PhabricatorFerretInterface')
19
+
->execute();
20
+
21
+
$did_collect = false;
22
+
foreach ($all_objects as $object) {
23
+
$engine = $object->newFerretEngine();
24
+
$conn = $object->establishConnection('w');
25
+
26
+
$ngram_row = queryfx_one(
27
+
$conn,
28
+
'SELECT ngram FROM %T WHERE needsCollection = 1 LIMIT 1',
29
+
$engine->getCommonNgramsTableName());
30
+
if (!$ngram_row) {
31
+
continue;
32
+
}
33
+
34
+
$ngram = $ngram_row['ngram'];
35
+
36
+
queryfx(
37
+
$conn,
38
+
'DELETE FROM %T WHERE ngram = %s',
39
+
$engine->getNgramsTableName(),
40
+
$ngram);
41
+
42
+
queryfx(
43
+
$conn,
44
+
'UPDATE %T SET needsCollection = 0 WHERE ngram = %s',
45
+
$engine->getCommonNgramsTableName(),
46
+
$ngram);
47
+
48
+
$did_collect = true;
49
+
break;
50
+
}
51
+
52
+
return $did_collect;
53
+
}
54
+
55
+
}
+4
-1
src/applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php
+4
-1
src/applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php
···
6
6
protected function didConstruct() {
7
7
$this
8
8
->setName('ngrams')
9
-
->setSynopsis(pht('Recompute common ngrams.'))
9
+
->setSynopsis(
10
+
pht(
11
+
'Recompute common ngrams. This is an advanced workflow that '.
12
+
'can harm search quality if used improperly.'))
10
13
->setArguments(
11
14
array(
12
15
array(