@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Parse raw Ferret queries into tokens before processing them
Summary:
Ref T12819. Depends on D18492. Instead of passing a raw query into the Query layer, parse it first.
This allows the query layer to figure out which parts should be substring vs term match, and would allow the SearchEngine layer to do `author:...` eventually by picking it out before sending it to the Ferret engine.
Test Plan: Ran some Ferret queries. They work like before, except that nonsense like `-+"quack"` raises an exception now.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T12819
Differential Revision: https://secure.phabricator.com/D18499
+32
-12
+16
-1
src/applications/maniphest/query/ManiphestTaskSearchEngine.php
+16
-1
src/applications/maniphest/query/ManiphestTaskSearchEngine.php
···
232
232
}
233
233
234
234
if (strlen($map['ferret'])) {
235
+
$raw_query = $map['ferret'];
236
+
237
+
$compiler = id(new PhutilSearchQueryCompiler())
238
+
->setEnableFunctions(true);
239
+
240
+
$raw_tokens = $compiler->newTokens($raw_query);
241
+
242
+
$fulltext_tokens = array();
243
+
foreach ($raw_tokens as $raw_token) {
244
+
$fulltext_token = id(new PhabricatorFulltextToken())
245
+
->setToken($raw_token);
246
+
247
+
$fulltext_tokens[] = $fulltext_token;
248
+
}
249
+
235
250
$query->withFerretConstraint(
236
251
id(new ManiphestTask())->newFerretEngine(),
237
-
$map['ferret']);
252
+
$fulltext_tokens);
238
253
}
239
254
240
255
if ($map['parentIDs']) {
+16
-11
src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php
+16
-11
src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php
···
28
28
private $spaceIsArchived;
29
29
private $ngrams = array();
30
30
private $ferretEngine;
31
-
private $ferretConstraints;
31
+
private $ferretTokens;
32
32
33
33
protected function getPageCursors(array $page) {
34
34
return array(
···
1386
1386
1387
1387
public function withFerretConstraint(
1388
1388
PhabricatorFerretEngine $engine,
1389
-
$raw_query) {
1389
+
array $fulltext_tokens) {
1390
1390
1391
1391
if ($this->ferretEngine) {
1392
1392
throw new Exception(
···
1394
1394
'Query may not have multiple fulltext constraints.'));
1395
1395
}
1396
1396
1397
-
if (!strlen($raw_query)) {
1397
+
if (!$fulltext_tokens) {
1398
1398
return $this;
1399
1399
}
1400
1400
1401
1401
$this->ferretEngine = $engine;
1402
-
$this->ferretConstraints = preg_split('/\s+/', $raw_query);
1402
+
$this->ferretTokens = $fulltext_tokens;
1403
1403
1404
1404
return $this;
1405
1405
}
···
1416
1416
$ngram_table_name = $ngram_table->getTableName();
1417
1417
1418
1418
$flat = array();
1419
-
foreach ($this->ferretConstraints as $term) {
1420
-
$value = $term;
1421
-
$length = count(phutil_utf8v($term));
1419
+
foreach ($this->ferretTokens as $fulltext_token) {
1420
+
$raw_token = $fulltext_token->getToken();
1421
+
$value = $raw_token->getValue();
1422
+
1423
+
$length = count(phutil_utf8v($value));
1422
1424
1423
1425
if ($length >= 3) {
1424
1426
$ngrams = $ngram_engine->getNgramsFromString($value, 'query');
···
1509
1511
}
1510
1512
1511
1513
$where = array();
1512
-
foreach ($this->ferretConstraints as $constraint) {
1514
+
foreach ($this->ferretTokens as $fulltext_token) {
1515
+
$raw_token = $fulltext_token->getToken();
1516
+
$value = $raw_token->getValue();
1517
+
1513
1518
$where[] = qsprintf(
1514
1519
$conn,
1515
1520
'(ftfield.rawCorpus LIKE %~ OR ftfield.normalCorpus LIKE %~)',
1516
-
$constraint,
1517
-
$constraint);
1521
+
$value,
1522
+
$value);
1518
1523
}
1519
1524
1520
1525
return $where;
1521
1526
}
1522
1527
1523
1528
protected function shouldGroupFerretResultRows() {
1524
-
return (bool)$this->ferretConstraints;
1529
+
return (bool)$this->ferretTokens;
1525
1530
}
1526
1531
1527
1532