@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Modularize Ferret fulltext functions
Summary: Ref T13511. Currently, Ferret fulltext field functions (like "title:") are hard-coded. Modularize them so extensions may define new ones.
Test Plan: Added a new custom field which emits data for the indexer, searched for "animal-noises:moo", "animal-noises:-", etc., in global search and application search.
Maniphest Tasks: T13511
Differential Revision: https://secure.phabricator.com/D21131
+261
-32
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
1288
1288
'FeedPushWorker' => 'applications/feed/worker/FeedPushWorker.php',
1289
1289
'FeedQueryConduitAPIMethod' => 'applications/feed/conduit/FeedQueryConduitAPIMethod.php',
1290
1290
'FeedStoryNotificationGarbageCollector' => 'applications/notification/garbagecollector/FeedStoryNotificationGarbageCollector.php',
1291
+
'FerretConfigurableSearchFunction' => 'applications/search/ferret/function/FerretConfigurableSearchFunction.php',
1292
+
'FerretSearchFunction' => 'applications/search/ferret/function/FerretSearchFunction.php',
1291
1293
'FileAllocateConduitAPIMethod' => 'applications/files/conduit/FileAllocateConduitAPIMethod.php',
1292
1294
'FileConduitAPIMethod' => 'applications/files/conduit/FileConduitAPIMethod.php',
1293
1295
'FileCreateMailReceiver' => 'applications/files/mail/FileCreateMailReceiver.php',
···
7402
7404
'FeedPushWorker' => 'PhabricatorWorker',
7403
7405
'FeedQueryConduitAPIMethod' => 'FeedConduitAPIMethod',
7404
7406
'FeedStoryNotificationGarbageCollector' => 'PhabricatorGarbageCollector',
7407
+
'FerretConfigurableSearchFunction' => 'FerretSearchFunction',
7408
+
'FerretSearchFunction' => 'Phobject',
7405
7409
'FileAllocateConduitAPIMethod' => 'FileConduitAPIMethod',
7406
7410
'FileConduitAPIMethod' => 'ConduitAPIMethod',
7407
7411
'FileCreateMailReceiver' => 'PhabricatorApplicationMailReceiver',
+1
-1
src/applications/search/compiler/PhutilSearchQueryCompiler.php
+1
-1
src/applications/search/compiler/PhutilSearchQueryCompiler.php
+8
src/applications/search/compiler/__tests__/PhutilSearchQueryCompilerTestCase.php
+8
src/applications/search/compiler/__tests__/PhutilSearchQueryCompilerTestCase.php
···
197
197
// impossible.
198
198
'title:- title:x' => false,
199
199
'title:- title:~' => false,
200
+
201
+
'abcdefghijklmnopqrstuvwxyz-ABCDEFGHIJKLMNOPQRSTUVWXYZ:xyz' => array(
202
+
array(
203
+
'abcdefghijklmnopqrstuvwxyz-ABCDEFGHIJKLMNOPQRSTUVWXYZ',
204
+
$op_and,
205
+
'xyz',
206
+
),
207
+
),
200
208
);
201
209
202
210
$this->assertCompileFunctionQueries($function_tests);
+20
src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php
+20
src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php
···
253
253
$old_id);
254
254
}
255
255
256
+
public function newFerretSearchFunctions() {
257
+
return array(
258
+
id(new FerretConfigurableSearchFunction())
259
+
->setFerretFunctionName('all')
260
+
->setFerretFieldKey(PhabricatorSearchDocumentFieldType::FIELD_ALL),
261
+
id(new FerretConfigurableSearchFunction())
262
+
->setFerretFunctionName('title')
263
+
->setFerretFieldKey(PhabricatorSearchDocumentFieldType::FIELD_TITLE),
264
+
id(new FerretConfigurableSearchFunction())
265
+
->setFerretFunctionName('body')
266
+
->setFerretFieldKey(PhabricatorSearchDocumentFieldType::FIELD_BODY),
267
+
id(new FerretConfigurableSearchFunction())
268
+
->setFerretFunctionName('core')
269
+
->setFerretFieldKey(PhabricatorSearchDocumentFieldType::FIELD_CORE),
270
+
id(new FerretConfigurableSearchFunction())
271
+
->setFerretFunctionName('comment')
272
+
->setFerretFieldKey(PhabricatorSearchDocumentFieldType::FIELD_COMMENT),
273
+
);
274
+
}
275
+
256
276
}
+22
-26
src/applications/search/ferret/PhabricatorFerretEngine.php
+22
-26
src/applications/search/ferret/PhabricatorFerretEngine.php
···
2
2
3
3
abstract class PhabricatorFerretEngine extends Phobject {
4
4
5
+
private $fieldMap = array();
6
+
private $ferretFunctions;
7
+
private $templateObject;
8
+
5
9
abstract public function getApplicationName();
6
10
abstract public function getScopeName();
7
11
abstract public function newSearchEngine();
···
14
18
return 1000;
15
19
}
16
20
17
-
public function getFieldForFunction($function) {
18
-
$function = phutil_utf8_strtolower($function);
21
+
final public function getFunctionForName($raw_name) {
22
+
if (isset($this->fieldMap[$raw_name])) {
23
+
return $this->fieldMap[$raw_name];
24
+
}
19
25
20
-
$map = $this->getFunctionMap();
21
-
if (!isset($map[$function])) {
26
+
$normalized_name =
27
+
FerretSearchFunction::getNormalizedFunctionName($raw_name);
28
+
29
+
if ($this->ferretFunctions === null) {
30
+
$functions = FerretSearchFunction::newFerretSearchFunctions();
31
+
$this->ferretFunctions = $functions;
32
+
}
33
+
34
+
if (!isset($this->ferretFunctions[$normalized_name])) {
22
35
throw new PhutilSearchQueryCompilerSyntaxException(
23
36
pht(
24
37
'Unknown search function "%s". Supported functions are: %s.',
25
-
$function,
26
-
implode(', ', array_keys($map))));
38
+
$raw_name,
39
+
implode(', ', array_keys($this->ferretFunctions))));
27
40
}
28
41
29
-
return $map[$function]['field'];
30
-
}
42
+
$function = $this->ferretFunctions[$normalized_name];
43
+
$this->fieldMap[$raw_name] = $function;
31
44
32
-
private function getFunctionMap() {
33
-
return array(
34
-
'all' => array(
35
-
'field' => PhabricatorSearchDocumentFieldType::FIELD_ALL,
36
-
),
37
-
'title' => array(
38
-
'field' => PhabricatorSearchDocumentFieldType::FIELD_TITLE,
39
-
),
40
-
'body' => array(
41
-
'field' => PhabricatorSearchDocumentFieldType::FIELD_BODY,
42
-
),
43
-
'core' => array(
44
-
'field' => PhabricatorSearchDocumentFieldType::FIELD_CORE,
45
-
),
46
-
'comment' => array(
47
-
'field' => PhabricatorSearchDocumentFieldType::FIELD_COMMENT,
48
-
),
49
-
);
45
+
return $this->fieldMap[$raw_name];
50
46
}
51
47
52
48
public function newStemmer() {
+31
src/applications/search/ferret/function/FerretConfigurableSearchFunction.php
+31
src/applications/search/ferret/function/FerretConfigurableSearchFunction.php
···
1
+
<?php
2
+
3
+
final class FerretConfigurableSearchFunction
4
+
extends FerretSearchFunction {
5
+
6
+
private $ferretFunctionName;
7
+
private $ferretFieldKey;
8
+
9
+
public function supportsObject(PhabricatorFerretInterface $object) {
10
+
return true;
11
+
}
12
+
13
+
public function setFerretFunctionName($ferret_function_name) {
14
+
$this->ferretFunctionName = $ferret_function_name;
15
+
return $this;
16
+
}
17
+
18
+
public function getFerretFunctionName() {
19
+
return $this->ferretFunctionName;
20
+
}
21
+
22
+
public function setFerretFieldKey($ferret_field_key) {
23
+
$this->ferretFieldKey = $ferret_field_key;
24
+
return $this;
25
+
}
26
+
27
+
public function getFerretFieldKey() {
28
+
return $this->ferretFieldKey;
29
+
}
30
+
31
+
}
+122
src/applications/search/ferret/function/FerretSearchFunction.php
+122
src/applications/search/ferret/function/FerretSearchFunction.php
···
1
+
<?php
2
+
3
+
abstract class FerretSearchFunction
4
+
extends Phobject {
5
+
6
+
abstract public function getFerretFunctionName();
7
+
abstract public function getFerretFieldKey();
8
+
abstract public function supportsObject(PhabricatorFerretInterface $object);
9
+
10
+
final public static function getNormalizedFunctionName($name) {
11
+
return phutil_utf8_strtolower($name);
12
+
}
13
+
14
+
final public static function validateFerretFunctionName($function_name) {
15
+
if (!preg_match('/^[a-zA-Z-]+\z/', $function_name)) {
16
+
throw new Exception(
17
+
pht(
18
+
'Ferret search engine function name ("%s") is invalid. Function '.
19
+
'names must be nonempty and may only contain latin letters and '.
20
+
'hyphens.'));
21
+
}
22
+
}
23
+
24
+
final public static function validateFerretFunctionFieldKey($field_key) {
25
+
if (!preg_match('/^[a-z]{4}\z/', $field_key)) {
26
+
throw new Exception(
27
+
pht(
28
+
'Ferret search engine field key ("%s") is invalid. Field keys '.
29
+
'must be exactly four characters long and contain only '.
30
+
'lowercase latin letters.',
31
+
$field_key));
32
+
}
33
+
}
34
+
35
+
final public static function newFerretSearchFunctions() {
36
+
$extensions = PhabricatorFulltextEngineExtension::getAllExtensions();
37
+
38
+
$function_map = array();
39
+
$field_map = array();
40
+
$results = array();
41
+
42
+
foreach ($extensions as $extension) {
43
+
$functions = $extension->newFerretSearchFunctions();
44
+
45
+
if (!is_array($functions)) {
46
+
throw new Exception(
47
+
pht(
48
+
'Expected fulltext engine extension ("%s") to return a '.
49
+
'list of functions from "newFerretSearchFunctions()", '.
50
+
'got "%s".',
51
+
get_class($extension),
52
+
phutil_describe_type($functions)));
53
+
}
54
+
55
+
foreach ($functions as $idx => $function) {
56
+
if (!($function instanceof FerretSearchFunction)) {
57
+
throw new Exception(
58
+
pht(
59
+
'Expected fulltext engine extension ("%s") to return a list '.
60
+
'of "FerretSearchFunction" objects from '.
61
+
'"newFerretSearchFunctions()", but found something else '.
62
+
'("%s") at index "%s".',
63
+
get_class($extension),
64
+
phutil_describe_type($function),
65
+
$idx));
66
+
}
67
+
68
+
$function_name = $function->getFerretFunctionName();
69
+
70
+
self::validateFerretFunctionName($function_name);
71
+
72
+
$normal_name = self::getNormalizedFunctionName(
73
+
$function_name);
74
+
if ($normal_name !== $function_name) {
75
+
throw new Exception(
76
+
pht(
77
+
'Ferret function "%s" is specified with a denormalized name. '.
78
+
'Instead, specify the function using the normalized '.
79
+
'function name ("%s").',
80
+
$normal_name));
81
+
}
82
+
83
+
if (isset($function_map[$function_name])) {
84
+
$other_extension = $function_map[$function_name];
85
+
throw new Exception(
86
+
pht(
87
+
'Two different fulltext engine extensions ("%s" and "%s") '.
88
+
'both define a search function with the same name ("%s"). '.
89
+
'Each function must have a unique name.',
90
+
get_class($extension),
91
+
get_class($other_extension),
92
+
$function_name));
93
+
}
94
+
$function_map[$function_name] = $extension;
95
+
96
+
$field_key = $function->getFerretFieldKey();
97
+
98
+
self::validateFerretFunctionFieldKey($field_key);
99
+
100
+
if (isset($field_map[$field_key])) {
101
+
$other_extension = $field_map[$field_key];
102
+
throw new Exception(
103
+
pht(
104
+
'Two different fulltext engine extensions ("%s" and "%s") '.
105
+
'both define a search function with the same key ("%s"). '.
106
+
'Each function must have a unique key.',
107
+
get_class($extension),
108
+
get_class($other_extension),
109
+
$field_key));
110
+
}
111
+
$field_map[$field_key] = $extension;
112
+
113
+
$results[$function_name] = $function;
114
+
}
115
+
}
116
+
117
+
ksort($results);
118
+
119
+
return $results;
120
+
}
121
+
122
+
}
+4
src/applications/search/index/PhabricatorFulltextEngineExtension.php
+4
src/applications/search/index/PhabricatorFulltextEngineExtension.php
+15
-1
src/applications/search/query/PhabricatorFulltextToken.php
+15
-1
src/applications/search/query/PhabricatorFulltextToken.php
···
43
43
$tip = null;
44
44
$icon = null;
45
45
46
+
$name = $token->getValue();
47
+
$function = $token->getFunction();
48
+
if ($function !== null) {
49
+
$name = pht('%s: %s', $function, $name);
50
+
}
51
+
46
52
if ($this->getIsShort()) {
47
53
$shade = PHUITagView::COLOR_GREY;
48
54
$tip = pht('Ignored Short Word');
···
64
70
$tip = pht('Exact Search');
65
71
$shade = PHUITagView::COLOR_GREEN;
66
72
break;
73
+
case PhutilSearchQueryCompiler::OPERATOR_PRESENT:
74
+
$name = pht('Field Present: %s', $function);
75
+
$shade = PHUITagView::COLOR_GREEN;
76
+
break;
77
+
case PhutilSearchQueryCompiler::OPERATOR_ABSENT:
78
+
$name = pht('Field Absent: %s', $function);
79
+
$shade = PHUITagView::COLOR_RED;
80
+
break;
67
81
default:
68
82
$shade = PHUITagView::COLOR_BLUE;
69
83
break;
···
73
87
$tag = id(new PHUITagView())
74
88
->setType(PHUITagView::TYPE_SHADE)
75
89
->setColor($shade)
76
-
->setName($token->getValue());
90
+
->setName($name);
77
91
78
92
if ($tip !== null) {
79
93
Javelin::initBehavior('phabricator-tooltips');
+34
-4
src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php
+34
-4
src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php
···
1815
1815
$function = $default_function;
1816
1816
}
1817
1817
1818
-
$raw_field = $engine->getFieldForFunction($function);
1818
+
$function_def = $engine->getFunctionForName($function);
1819
1819
1820
1820
// NOTE: The query compiler guarantees that a query can not make a
1821
1821
// field both "present" and "absent", so it's safe to just use the
···
1829
1829
$alias = 'ftfield_'.$idx++;
1830
1830
$table_map[$function] = array(
1831
1831
'alias' => $alias,
1832
-
'key' => $raw_field,
1832
+
'function' => $function_def,
1833
1833
'optional' => $is_optional,
1834
1834
);
1835
1835
}
···
1838
1838
// Join the title field separately so we can rank results.
1839
1839
$table_map['rank'] = array(
1840
1840
'alias' => 'ft_rank',
1841
-
'key' => PhabricatorSearchDocumentFieldType::FIELD_TITLE,
1841
+
'function' => $engine->getFunctionForName('title'),
1842
1842
1843
1843
// See T13345. Not every document has a title, so we want to LEFT JOIN
1844
1844
// this table to avoid excluding documents with no title that match
···
2130
2130
$ngram);
2131
2131
}
2132
2132
2133
+
$object = $this->newResultObject();
2134
+
if (!$object) {
2135
+
throw new Exception(
2136
+
pht(
2137
+
'Query class ("%s") must define "newResultObject()" to use '.
2138
+
'Ferret constraints.',
2139
+
get_class($this)));
2140
+
}
2141
+
2142
+
// See T13511. If we have a fulltext query which uses valid field
2143
+
// functions, but at least one of the functions applies to a field which
2144
+
// the object can never have, the query can never match anything. Detect
2145
+
// this and return an empty result set.
2146
+
2147
+
// (Even if the query is "field is absent" or "field does not contain
2148
+
// such-and-such", the interpretation is that these constraints are
2149
+
// not meaningful when applied to an object which can never have the
2150
+
// field.)
2151
+
2152
+
$functions = ipull($this->ferretTables, 'function');
2153
+
$functions = mpull($functions, null, 'getFerretFunctionName');
2154
+
foreach ($functions as $function) {
2155
+
if (!$function->supportsObject($object)) {
2156
+
throw new PhabricatorEmptyQueryException(
2157
+
pht(
2158
+
'This query uses a fulltext function which this document '.
2159
+
'type does not support.'));
2160
+
}
2161
+
}
2162
+
2133
2163
foreach ($this->ferretTables as $table) {
2134
2164
$alias = $table['alias'];
2135
2165
···
2148
2178
$alias,
2149
2179
$alias,
2150
2180
$alias,
2151
-
$table['key']);
2181
+
$table['function']->getFerretFieldKey());
2152
2182
}
2153
2183
2154
2184
return $joins;