@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Separate sever-side typeahead queries into "prefix" and "content" phases
Summary:
Ref T8510. When users type "platypus" into a typeahead, they want "Platypus Playground" to be a higher-ranked match than "AAA Platypus", even though the latter is alphabetically first.
Specifically, the rule is: results which match the query as a prefix of the result text should rank above results which do not.
I believe we now always get this right on the client side. However, WMF has at least one case (described in T8510) where we do not get it right on the server side, and thus the user sees the wrong result.
The remaining issue is that if "platypus" matches more than 100 results, the result "Platypus Playground" may not appear in the result set at all, beacuse there are 100 copies of "AAA Platypus 1", "AAA Platypus 2", etc., first. So even though the client will apply the correct sort, it doesn't have the result the user wants and can't show it to them.
To fix this, split the server-side query into two phases:
- In the first phase, the "prefix" phase, we find results that **start with** "platypus".
- In the second phase, the "content" phase, we find results that contain "platypus" anywhere.
We skip the "prefix" phase if the user has not typed a query (for example, in the browse view).
Test Plan:
This is a lot of stuff, but the new ranking here puts projects which start with "w" at the top of the list. Lower down the list, you can see some projects which contain "w" but do not appear at the top (like "Serious Work").
{F1913931}
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T8510
Differential Revision: https://secure.phabricator.com/D16838
+219
-11
+17
src/applications/people/query/PhabricatorPeopleQuery.php
+17
src/applications/people/query/PhabricatorPeopleQuery.php
···
17
17
private $isApproved;
18
18
private $nameLike;
19
19
private $nameTokens;
20
+
private $namePrefixes;
20
21
21
22
private $needPrimaryEmail;
22
23
private $needProfile;
···
92
93
93
94
public function withNameTokens(array $tokens) {
94
95
$this->nameTokens = array_values($tokens);
96
+
return $this;
97
+
}
98
+
99
+
public function withNamePrefixes(array $prefixes) {
100
+
$this->namePrefixes = $prefixes;
95
101
return $this;
96
102
}
97
103
···
254
260
$conn,
255
261
'user.userName IN (%Ls)',
256
262
$this->usernames);
263
+
}
264
+
265
+
if ($this->namePrefixes) {
266
+
$parts = array();
267
+
foreach ($this->namePrefixes as $name_prefix) {
268
+
$parts[] = qsprintf(
269
+
$conn,
270
+
'user.username LIKE %>',
271
+
$name_prefix);
272
+
}
273
+
$where[] = '('.implode(' OR ', $parts).')';
257
274
}
258
275
259
276
if ($this->emails !== null) {
+8
-3
src/applications/people/typeahead/PhabricatorPeopleDatasource.php
+8
-3
src/applications/people/typeahead/PhabricatorPeopleDatasource.php
···
17
17
18
18
public function loadResults() {
19
19
$viewer = $this->getViewer();
20
-
$tokens = $this->getTokens();
21
20
22
21
$query = id(new PhabricatorPeopleQuery())
23
22
->setOrderVector(array('username'));
24
23
25
-
if ($tokens) {
26
-
$query->withNameTokens($tokens);
24
+
if ($this->getPhase() == self::PHASE_PREFIX) {
25
+
$prefix = $this->getPrefixQuery();
26
+
$query->withNamePrefixes(array($prefix));
27
+
} else {
28
+
$tokens = $this->getTokens();
29
+
if ($tokens) {
30
+
$query->withNameTokens($tokens);
31
+
}
27
32
}
28
33
29
34
$users = $this->executeQuery($query);
+17
src/applications/project/query/PhabricatorProjectQuery.php
+17
src/applications/project/query/PhabricatorProjectQuery.php
···
12
12
private $slugMap;
13
13
private $allSlugs;
14
14
private $names;
15
+
private $namePrefixes;
15
16
private $nameTokens;
16
17
private $icons;
17
18
private $colors;
···
75
76
76
77
public function withNames(array $names) {
77
78
$this->names = $names;
79
+
return $this;
80
+
}
81
+
82
+
public function withNamePrefixes(array $prefixes) {
83
+
$this->namePrefixes = $prefixes;
78
84
return $this;
79
85
}
80
86
···
462
468
$conn,
463
469
'name IN (%Ls)',
464
470
$this->names);
471
+
}
472
+
473
+
if ($this->namePrefixes) {
474
+
$parts = array();
475
+
foreach ($this->namePrefixes as $name_prefix) {
476
+
$parts[] = qsprintf(
477
+
$conn,
478
+
'name LIKE %>',
479
+
$name_prefix);
480
+
}
481
+
$where[] = '('.implode(' OR ', $parts).')';
465
482
}
466
483
467
484
if ($this->icons !== null) {
+4
-1
src/applications/project/typeahead/PhabricatorProjectDatasource.php
+4
-1
src/applications/project/typeahead/PhabricatorProjectDatasource.php
+5
src/applications/typeahead/controller/PhabricatorTypeaheadModularDatasourceController.php
+5
src/applications/typeahead/controller/PhabricatorTypeaheadModularDatasourceController.php
+133
-7
src/applications/typeahead/datasource/PhabricatorTypeaheadCompositeDatasource.php
+133
-7
src/applications/typeahead/datasource/PhabricatorTypeaheadCompositeDatasource.php
···
4
4
extends PhabricatorTypeaheadDatasource {
5
5
6
6
private $usable;
7
+
private $prefixString;
8
+
private $prefixLength;
7
9
8
10
abstract public function getComponentDatasources();
9
11
···
22
24
}
23
25
24
26
public function loadResults() {
27
+
$phases = array();
28
+
29
+
// We only need to do a prefix phase query if there's an actual query
30
+
// string. If the user didn't type anything, nothing can possibly match it.
31
+
if (strlen($this->getRawQuery())) {
32
+
$phases[] = self::PHASE_PREFIX;
33
+
}
34
+
35
+
$phases[] = self::PHASE_CONTENT;
36
+
25
37
$offset = $this->getOffset();
26
38
$limit = $this->getLimit();
27
39
40
+
$results = array();
41
+
foreach ($phases as $phase) {
42
+
if ($limit) {
43
+
$phase_limit = ($offset + $limit) - count($results);
44
+
} else {
45
+
$phase_limit = 0;
46
+
}
47
+
48
+
$phase_results = $this->loadResultsForPhase(
49
+
$phase,
50
+
$phase_limit);
51
+
52
+
foreach ($phase_results as $result) {
53
+
$results[] = $result;
54
+
}
55
+
56
+
if ($limit) {
57
+
if (count($results) >= $offset + $limit) {
58
+
break;
59
+
}
60
+
}
61
+
}
62
+
63
+
return $results;
64
+
}
65
+
66
+
protected function loadResultsForPhase($phase, $limit) {
67
+
if ($phase == self::PHASE_PREFIX) {
68
+
$this->prefixString = $this->getPrefixQuery();
69
+
$this->prefixLength = strlen($this->prefixString);
70
+
}
71
+
28
72
// If the input query is a function like `members(platy`, and we can
29
73
// parse the function, we strip the function off and hand the stripped
30
74
// query to child sources. This makes it easier to implement function
···
62
106
}
63
107
64
108
$source
109
+
->setPhase($phase)
65
110
->setFunctionStack($source_stack)
66
111
->setRawQuery($source_query)
67
112
->setQuery($this->getQuery())
68
113
->setViewer($this->getViewer());
69
-
70
-
if ($limit) {
71
-
$source->setLimit($offset + $limit);
72
-
}
73
114
74
115
if ($is_browse) {
75
116
$source->setIsBrowse(true);
76
117
}
77
118
78
-
$source_results = $source->loadResults();
79
-
$source_results = $source->didLoadResults($source_results);
119
+
if ($limit) {
120
+
// If we are loading results from a source with a limit, it may return
121
+
// some results which belong to the wrong phase. We need an entire page
122
+
// of valid results in the correct phase AFTER any results for the
123
+
// wrong phase are filtered for pagination to work correctly.
124
+
125
+
// To make sure we can get there, we fetch more and more results until
126
+
// enough of them survive filtering to generate a full page.
127
+
128
+
// We start by fetching 150% of the results than we think we need, and
129
+
// double the amount we overfetch by each time.
130
+
$factor = 1.5;
131
+
while (true) {
132
+
$query_source = clone $source;
133
+
$total = (int)ceil($limit * $factor) + 1;
134
+
$query_source->setLimit($total);
135
+
136
+
$source_results = $query_source->loadResultsForPhase(
137
+
$phase,
138
+
$limit);
139
+
140
+
// If there are fewer unfiltered results than we asked for, we know
141
+
// this is the entire result set and we don't need to keep going.
142
+
if (count($source_results) < $total) {
143
+
$source_results = $query_source->didLoadResults($source_results);
144
+
$source_results = $this->filterPhaseResults(
145
+
$phase,
146
+
$source_results);
147
+
break;
148
+
}
149
+
150
+
// Otherwise, this result set have everything we need, or may not.
151
+
// Filter the results that are part of the wrong phase out first...
152
+
$source_results = $query_source->didLoadResults($source_results);
153
+
$source_results = $this->filterPhaseResults($phase, $source_results);
154
+
155
+
// Now check if we have enough results left. If we do, we're all set.
156
+
if (count($source_results) >= $total) {
157
+
break;
158
+
}
159
+
160
+
// We filtered out too many results to have a full page left, so we
161
+
// need to run the query again, asking for even more results. We'll
162
+
// keep doing this until we get a full page or get all of the
163
+
// results.
164
+
$factor = $factor * 2;
165
+
}
166
+
} else {
167
+
$source_results = $source->loadResults();
168
+
$source_results = $source->didLoadResults($source_results);
169
+
$source_results = $this->filterPhaseResults($phase, $source_results);
170
+
}
171
+
80
172
$results[] = $source_results;
81
173
}
82
174
83
175
$results = array_mergev($results);
84
176
$results = msort($results, 'getSortKey');
85
177
86
-
$count = count($results);
178
+
$results = $this->sliceResults($results);
179
+
180
+
return $results;
181
+
}
182
+
183
+
private function filterPhaseResults($phase, $source_results) {
184
+
foreach ($source_results as $key => $source_result) {
185
+
$result_phase = $this->getResultPhase($source_result);
186
+
187
+
if ($result_phase != $phase) {
188
+
unset($source_results[$key]);
189
+
continue;
190
+
}
191
+
192
+
$source_result->setPhase($result_phase);
193
+
}
194
+
195
+
return $source_results;
196
+
}
197
+
198
+
private function getResultPhase(PhabricatorTypeaheadResult $result) {
199
+
if ($this->prefixLength) {
200
+
$result_name = phutil_utf8_strtolower($result->getName());
201
+
if (!strncmp($result_name, $this->prefixString, $this->prefixLength)) {
202
+
return self::PHASE_PREFIX;
203
+
}
204
+
}
205
+
206
+
return self::PHASE_CONTENT;
207
+
}
208
+
209
+
protected function sliceResults(array $results) {
210
+
$offset = $this->getOffset();
211
+
$limit = $this->getLimit();
212
+
87
213
if ($offset || $limit) {
88
214
if (!$limit) {
89
215
$limit = count($results);
+24
src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php
+24
src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php
···
13
13
private $parameters = array();
14
14
private $functionStack = array();
15
15
private $isBrowse;
16
+
private $phase = self::PHASE_CONTENT;
17
+
18
+
const PHASE_PREFIX = 'prefix';
19
+
const PHASE_CONTENT = 'content';
16
20
17
21
public function setLimit($limit) {
18
22
$this->limit = $limit;
···
46
50
return $this;
47
51
}
48
52
53
+
public function getPrefixQuery() {
54
+
return phutil_utf8_strtolower($this->getRawQuery());
55
+
}
56
+
49
57
public function getRawQuery() {
50
58
return $this->rawQuery;
51
59
}
···
81
89
return $this->isBrowse;
82
90
}
83
91
92
+
public function setPhase($phase) {
93
+
$this->phase = $phase;
94
+
return $this;
95
+
}
96
+
97
+
public function getPhase() {
98
+
return $this->phase;
99
+
}
100
+
84
101
public function getDatasourceURI() {
85
102
$uri = new PhutilURI('/typeahead/class/'.get_class($this).'/');
86
103
$uri->setQueryParams($this->parameters);
···
105
122
106
123
abstract public function getDatasourceApplicationClass();
107
124
abstract public function loadResults();
125
+
126
+
protected function loadResultsForPhase($phase, $limit) {
127
+
// By default, sources just load all of their results in every phase and
128
+
// rely on filtering at a higher level to sequence phases correctly.
129
+
$this->setLimit($limit);
130
+
return $this->loadResults();
131
+
}
108
132
109
133
protected function didLoadResults(array $results) {
110
134
return $results;
+11
src/applications/typeahead/storage/PhabricatorTypeaheadResult.php
+11
src/applications/typeahead/storage/PhabricatorTypeaheadResult.php
···
18
18
private $unique;
19
19
private $autocomplete;
20
20
private $attributes = array();
21
+
private $phase;
21
22
22
23
public function setIcon($icon) {
23
24
$this->icon = $icon;
···
154
155
$this->tokenType,
155
156
$this->unique ? 1 : null,
156
157
$this->autocomplete,
158
+
$this->phase,
157
159
);
158
160
while (end($data) === null) {
159
161
array_pop($data);
···
209
211
public function addAttribute($attribute) {
210
212
$this->attributes[] = $attribute;
211
213
return $this;
214
+
}
215
+
216
+
public function setPhase($phase) {
217
+
$this->phase = $phase;
218
+
return $this;
219
+
}
220
+
221
+
public function getPhase() {
222
+
return $this->phase;
212
223
}
213
224
214
225
}