@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Use the Ferret engine fulltext document table to drive auxiliary fulltext constraints
Summary:
Ref T12819. I started trying to get individual engines to drive these constraints (e.g., `ManiphestTaskQuery` can do most of the work) but this is a big pain, especially since most engines don't support "any owner" or "no owner", and not everything has an owner, and so on and so on. Going down this path would have meant a huge pile of stub functions everywhere, I think.
Instead, drive these through the main engine using the fulltext document table, which already has everything we need to apply these constraints in a uniform way.
Also tweak some parts of query construction and result ordering.
Test Plan: Searched for documents by author, owner, unowned, any owner, tags, subscribers, fulltext in global search. Got sensible results without any application-specific code.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T12819
Differential Revision: https://secure.phabricator.com/D18550
+171
-77
+1
-1
src/applications/differential/search/DifferentialRevisionFerretEngine.php
+1
-1
src/applications/differential/search/DifferentialRevisionFerretEngine.php
+1
-1
src/applications/maniphest/search/ManiphestTaskFerretEngine.php
+1
-1
src/applications/maniphest/search/ManiphestTaskFerretEngine.php
+29
-3
src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php
+29
-3
src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php
···
23
23
$phid = $document->getPHID();
24
24
$engine = $object->newFerretEngine();
25
25
26
+
$is_closed = 0;
27
+
$author_phid = null;
28
+
$owner_phid = null;
29
+
foreach ($document->getRelationshipData() as $relationship) {
30
+
list($related_type, $related_phid) = $relationship;
31
+
switch ($related_type) {
32
+
case PhabricatorSearchRelationship::RELATIONSHIP_OPEN:
33
+
$is_closed = 0;
34
+
break;
35
+
case PhabricatorSearchRelationship::RELATIONSHIP_CLOSED:
36
+
$is_closed = 1;
37
+
break;
38
+
case PhabricatorSearchRelationship::RELATIONSHIP_OWNER:
39
+
$owner_phid = $related_phid;
40
+
break;
41
+
case PhabricatorSearchRelationship::RELATIONSHIP_UNOWNED:
42
+
$owner_phid = null;
43
+
break;
44
+
case PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR:
45
+
$author_phid = $related_phid;
46
+
break;
47
+
}
48
+
}
49
+
26
50
$ferret_document = $engine->newDocumentObject()
27
51
->setObjectPHID($phid)
28
-
->setIsClosed(0)
29
-
->setEpochCreated(0)
30
-
->setEpochModified(0);
52
+
->setIsClosed($is_closed)
53
+
->setEpochCreated($document->getDocumentCreated())
54
+
->setEpochModified($document->getDocumentModified())
55
+
->setAuthorPHID($author_phid)
56
+
->setOwnerPHID($owner_phid);
31
57
32
58
$stemmer = $engine->newStemmer();
33
59
+12
src/applications/search/ferret/PhabricatorFerretDocument.php
+12
src/applications/search/ferret/PhabricatorFerretDocument.php
···
27
27
'columns' => array('objectPHID'),
28
28
'unique' => true,
29
29
),
30
+
'key_author' => array(
31
+
'columns' => array('authorPHID'),
32
+
),
33
+
'key_owner' => array(
34
+
'columns' => array('ownerPHID'),
35
+
),
36
+
'key_created' => array(
37
+
'columns' => array('epochCreated'),
38
+
),
39
+
'key_modified' => array(
40
+
'columns' => array('epochModified'),
41
+
),
30
42
),
31
43
) + parent::getConfiguration();
32
44
}
+1
-55
src/applications/search/ferret/PhabricatorFerretEngine.php
+1
-55
src/applications/search/ferret/PhabricatorFerretEngine.php
···
5
5
abstract public function newNgramsObject();
6
6
abstract public function newDocumentObject();
7
7
abstract public function newFieldObject();
8
-
abstract protected function newSearchEngine();
8
+
abstract public function newSearchEngine();
9
9
10
10
public function getDefaultFunctionKey() {
11
11
return 'all';
···
68
68
69
69
public function newStemmer() {
70
70
return new PhutilSearchStemmer();
71
-
}
72
-
73
-
public function newConfiguredFulltextQuery(
74
-
$object,
75
-
PhabricatorSavedQuery $query,
76
-
PhabricatorUser $viewer) {
77
-
78
-
$local_query = new PhabricatorSavedQuery();
79
-
$local_query->setParameter('query', $query->getParameter('query'));
80
-
81
-
// TODO: Modularize this piece.
82
-
$project_phids = $query->getParameter('projectPHIDs');
83
-
if ($project_phids) {
84
-
$local_query->setParameter('projectPHIDs', $project_phids);
85
-
}
86
-
87
-
$subscriber_phids = $query->getParameter('subscriberPHIDs');
88
-
if ($subscriber_phids) {
89
-
$local_query->setParameter('subscriberPHIDs', $subscriber_phids);
90
-
}
91
-
92
-
$author_phids = $query->getParameter('authorPHIDs');
93
-
if ($author_phids) {
94
-
$local_query->setParameter('authorPHIDs', $author_phids);
95
-
}
96
-
97
-
$owner_phids = $query->getParameter('ownerPHIDs');
98
-
if ($owner_phids) {
99
-
$local_query->setParameter('ownerPHIDs', $owner_phids);
100
-
}
101
-
102
-
$rel_open = PhabricatorSearchRelationship::RELATIONSHIP_OPEN;
103
-
$rel_closed = PhabricatorSearchRelationship::RELATIONSHIP_CLOSED;
104
-
105
-
$statuses = $query->getParameter('statuses');
106
-
if ($statuses) {
107
-
$statuses = array_fuse($statuses);
108
-
if (count($statuses) == 1) {
109
-
if (isset($statuses[$rel_open])) {
110
-
$local_query->setParameter('statuses', array('open()'));
111
-
}
112
-
if (isset($statuses[$rel_closed])) {
113
-
$local_query->setParameter('statuses', array('closed()'));
114
-
}
115
-
}
116
-
}
117
-
118
-
$search_engine = $this->newSearchEngine()
119
-
->setViewer($viewer);
120
-
121
-
$engine_query = $search_engine->buildQueryFromSavedQuery($local_query)
122
-
->setViewer($viewer);
123
-
124
-
return $engine_query;
125
71
}
126
72
127
73
public function tokenizeString($value) {
+22
-8
src/applications/search/fulltextstorage/PhabricatorFerretFulltextStorageEngine.php
+22
-8
src/applications/search/fulltextstorage/PhabricatorFerretFulltextStorageEngine.php
···
47
47
$offset = (int)$query->getParameter('offset', 0);
48
48
$limit = (int)$query->getParameter('limit', 25);
49
49
50
+
// NOTE: For now, it's okay to query with the omnipotent viewer here
51
+
// because we're just returning PHIDs which we'll filter later.
50
52
$viewer = PhabricatorUser::getOmnipotentUser();
51
53
52
54
$type_results = array();
···
54
56
$engine = $spec['engine'];
55
57
$object = $spec['object'];
56
58
57
-
// NOTE: For now, it's okay to query with the omnipotent viewer here
58
-
// because we're just returning PHIDs which we'll filter later.
59
+
$local_query = new PhabricatorSavedQuery();
60
+
$local_query->setParameter('query', $query->getParameter('query'));
59
61
60
-
$type_query = $engine->newConfiguredFulltextQuery(
61
-
$object,
62
-
$query,
63
-
$viewer);
62
+
$project_phids = $query->getParameter('projectPHIDs');
63
+
if ($project_phids) {
64
+
$local_query->setParameter('projectPHIDs', $project_phids);
65
+
}
64
66
65
-
$type_query
67
+
$subscriber_phids = $query->getParameter('subscriberPHIDs');
68
+
if ($subscriber_phids) {
69
+
$local_query->setParameter('subscriberPHIDs', $subscriber_phids);
70
+
}
71
+
72
+
$search_engine = $engine->newSearchEngine()
73
+
->setViewer($viewer);
74
+
75
+
$engine_query = $search_engine->buildQueryFromSavedQuery($local_query)
76
+
->setViewer($viewer);
77
+
78
+
$engine_query
79
+
->withFerretQuery($engine, $query)
66
80
->setOrder('relevance')
67
81
->setLimit($offset + $limit);
68
82
69
-
$results = $type_query->execute();
83
+
$results = $engine_query->execute();
70
84
$results = mpull($results, null, 'getPHID');
71
85
$type_results[$type] = $results;
72
86
}
+105
-9
src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php
+105
-9
src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php
···
28
28
private $spaceIsArchived;
29
29
private $ngrams = array();
30
30
private $ferretEngine;
31
-
private $ferretTokens;
32
-
private $ferretTables;
31
+
private $ferretTokens = array();
32
+
private $ferretTables = array();
33
+
private $ferretQuery;
33
34
34
35
protected function getPageCursors(array $page) {
35
36
return array(
···
772
773
773
774
if ($this->supportsFerretEngine()) {
774
775
$orders['relevance'] = array(
775
-
'vector' => array('rank', 'id'),
776
+
'vector' => array('rank', 'fulltext-modified', 'id'),
776
777
'name' => pht('Relevence'),
777
778
);
778
779
}
···
973
974
$columns['rank'] = array(
974
975
'table' => null,
975
976
'column' => '_ft_rank',
977
+
'type' => 'int',
978
+
);
979
+
$columns['fulltext-created'] = array(
980
+
'table' => 'ft_doc',
981
+
'column' => 'epochCreated',
982
+
'type' => 'int',
983
+
);
984
+
$columns['fulltext-modified'] = array(
985
+
'table' => 'ft_doc',
986
+
'column' => 'epochModified',
976
987
'type' => 'int',
977
988
);
978
989
}
···
1406
1417
return ($object instanceof PhabricatorFerretInterface);
1407
1418
}
1408
1419
1420
+
public function withFerretQuery(
1421
+
PhabricatorFerretEngine $engine,
1422
+
PhabricatorSavedQuery $query) {
1423
+
1424
+
if (!$this->supportsFerretEngine()) {
1425
+
throw new Exception(
1426
+
pht(
1427
+
'Query ("%s") does not support the Ferret fulltext engine.',
1428
+
get_class($this)));
1429
+
}
1430
+
1431
+
$this->ferretEngine = $engine;
1432
+
$this->ferretQuery = $query;
1433
+
1434
+
return $this;
1435
+
}
1409
1436
1410
1437
public function withFerretConstraint(
1411
1438
PhabricatorFerretEngine $engine,
···
1538
1565
$table_alias,
1539
1566
$stem_value);
1540
1567
}
1541
-
1542
-
$parts[] = '0';
1543
1568
}
1569
+
1570
+
$parts[] = '0';
1544
1571
1545
1572
$select[] = qsprintf(
1546
1573
$conn,
···
1646
1673
$joins = array();
1647
1674
$joins[] = qsprintf(
1648
1675
$conn,
1649
-
'JOIN %T ftdoc ON ftdoc.objectPHID = %Q',
1676
+
'JOIN %T ft_doc ON ft_doc.objectPHID = %Q',
1650
1677
$document_table->getTableName(),
1651
1678
$phid_column);
1652
1679
···
1655
1682
$table = $spec['table'];
1656
1683
$ngram = $spec['ngram'];
1657
1684
1658
-
$alias = 'ft'.$idx++;
1685
+
$alias = 'ftngram_'.$idx++;
1659
1686
1660
1687
$joins[] = qsprintf(
1661
1688
$conn,
1662
-
'JOIN %T %T ON %T.documentID = ftdoc.id AND %T.ngram = %s',
1689
+
'JOIN %T %T ON %T.documentID = ft_doc.id AND %T.ngram = %s',
1663
1690
$table,
1664
1691
$alias,
1665
1692
$alias,
···
1672
1699
1673
1700
$joins[] = qsprintf(
1674
1701
$conn,
1675
-
'JOIN %T %T ON ftdoc.id = %T.documentID
1702
+
'JOIN %T %T ON ft_doc.id = %T.documentID
1676
1703
AND %T.fieldKey = %s',
1677
1704
$field_table->getTableName(),
1678
1705
$alias,
···
1798
1825
$conn,
1799
1826
'(%Q)',
1800
1827
implode(' OR ', $term_constraints));
1828
+
}
1829
+
}
1830
+
1831
+
if ($this->ferretQuery) {
1832
+
$query = $this->ferretQuery;
1833
+
1834
+
$author_phids = $query->getParameter('authorPHIDs');
1835
+
if ($author_phids) {
1836
+
$where[] = qsprintf(
1837
+
$conn,
1838
+
'ft_doc.authorPHID IN (%Ls)',
1839
+
$author_phids);
1840
+
}
1841
+
1842
+
$with_unowned = $query->getParameter('withUnowned');
1843
+
$with_any = $query->getParameter('withAnyOwner');
1844
+
1845
+
if ($with_any && $with_unowned) {
1846
+
throw new PhabricatorEmptyQueryException(
1847
+
pht(
1848
+
'This query matches only unowned documents owned by anyone, '.
1849
+
'which is impossible.'));
1850
+
}
1851
+
1852
+
$owner_phids = $query->getParameter('ownerPHIDs');
1853
+
if ($owner_phids && !$with_any) {
1854
+
if ($with_unowned) {
1855
+
$where[] = qsprintf(
1856
+
$conn,
1857
+
'ft_doc.ownerPHID IN (%Ls) OR ft_doc.ownerPHID IS NULL',
1858
+
$owner_phids);
1859
+
} else {
1860
+
$where[] = qsprintf(
1861
+
$conn,
1862
+
'ft_doc.ownerPHID IN (%Ls)',
1863
+
$owner_phids);
1864
+
}
1865
+
} else if ($with_unowned) {
1866
+
$where[] = qsprintf(
1867
+
$conn,
1868
+
'ft_doc.ownerPHID IS NULL');
1869
+
}
1870
+
1871
+
if ($with_any) {
1872
+
$where[] = qsprintf(
1873
+
$conn,
1874
+
'ft_doc.ownerPHID IS NOT NULL');
1875
+
}
1876
+
1877
+
$rel_open = PhabricatorSearchRelationship::RELATIONSHIP_OPEN;
1878
+
1879
+
$statuses = $query->getParameter('statuses');
1880
+
$is_closed = null;
1881
+
if ($statuses) {
1882
+
$statuses = array_fuse($statuses);
1883
+
if (count($statuses) == 1) {
1884
+
if (isset($statuses[$rel_open])) {
1885
+
$is_closed = 0;
1886
+
} else {
1887
+
$is_closed = 1;
1888
+
}
1889
+
}
1890
+
}
1891
+
1892
+
if ($is_closed !== null) {
1893
+
$where[] = qsprintf(
1894
+
$conn,
1895
+
'ft_doc.isClosed = %d',
1896
+
$is_closed);
1801
1897
}
1802
1898
}
1803
1899