@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Extend PhabricatorPolicyQuery from PhabricatorOffsetPagedQuery
Summary:
A few goals here:
- Slightly simplify the Query classtree -- it's now linear: `Query` -> `OffsetPagedQuery` (adds offset/limit) -> `PolicyQuery` (adds policy filtering) -> `CursorPagedPolicyQuery` (adds cursors).
- Allow us to move from non-policy queries to policy queries without any backward compatibility breaks, e.g. Conduit methods which accept 'offset'.
- Separate the client limit ("limit") from the datafetch hint limit ("rawresultlimit") so we can make the heurstic smarter in the future if we want. Some discussion inline.
Test Plan: Expanded unit tests to cover offset behaviors.
Reviewers: vrana, btrahan
Reviewed By: btrahan
CC: aran
Maniphest Tasks: T603
Differential Revision: https://secure.phabricator.com/D3192
+250
-18
+34
src/applications/policy/__tests__/PhabricatorPolicyTestCase.php
+34
src/applications/policy/__tests__/PhabricatorPolicyTestCase.php
···
103
103
104
104
105
105
/**
106
+
* Test offset-based filtering.
107
+
*/
108
+
public function testOffsets() {
109
+
$results = array(
110
+
$this->buildObject(PhabricatorPolicies::POLICY_NOONE),
111
+
$this->buildObject(PhabricatorPolicies::POLICY_NOONE),
112
+
$this->buildObject(PhabricatorPolicies::POLICY_NOONE),
113
+
$this->buildObject(PhabricatorPolicies::POLICY_USER),
114
+
$this->buildObject(PhabricatorPolicies::POLICY_USER),
115
+
$this->buildObject(PhabricatorPolicies::POLICY_USER),
116
+
);
117
+
118
+
$query = new PhabricatorPolicyTestQuery();
119
+
$query->setResults($results);
120
+
$query->setViewer($this->buildUser('user'));
121
+
122
+
$this->assertEqual(
123
+
3,
124
+
count($query->setLimit(3)->setOffset(0)->execute()),
125
+
'Invisible objects are ignored.');
126
+
127
+
$this->assertEqual(
128
+
0,
129
+
count($query->setLimit(3)->setOffset(3)->execute()),
130
+
'Offset pages through visible objects only.');
131
+
132
+
$this->assertEqual(
133
+
2,
134
+
count($query->setLimit(3)->setOffset(1)->execute()),
135
+
'Offsets work correctly.');
136
+
}
137
+
138
+
139
+
/**
106
140
* Test an object for visibility across multiple user specifications.
107
141
*/
108
142
private function expectVisibility(
+14
-2
src/applications/policy/__tests__/PhabricatorPolicyTestQuery.php
+14
-2
src/applications/policy/__tests__/PhabricatorPolicyTestQuery.php
···
23
23
extends PhabricatorPolicyQuery {
24
24
25
25
private $results;
26
+
private $offset = 0;
26
27
27
28
public function setResults(array $results) {
28
29
$this->results = $results;
29
30
return $this;
30
31
}
31
32
33
+
protected function willExecute() {
34
+
$this->offset = 0;
35
+
}
36
+
32
37
public function loadPage() {
33
-
return $this->results;
38
+
if ($this->getRawResultLimit()) {
39
+
return array_slice(
40
+
$this->results,
41
+
$this->offset,
42
+
$this->getRawResultLimit());
43
+
} else {
44
+
return array_slice($this->results, $this->offset);
45
+
}
34
46
}
35
47
36
48
public function nextPage(array $page) {
37
-
return null;
49
+
$this->offset += count($page);
38
50
}
39
51
40
52
}
+9
-1
src/infrastructure/query/PhabricatorOffsetPagedQuery.php
+9
-1
src/infrastructure/query/PhabricatorOffsetPagedQuery.php
···
35
35
return $this;
36
36
}
37
37
38
-
final protected function buildLimitClause(AphrontDatabaseConnection $conn_r) {
38
+
final public function getOffset() {
39
+
return $this->offset;
40
+
}
41
+
42
+
final public function getLimit() {
43
+
return $this->limit;
44
+
}
45
+
46
+
protected function buildLimitClause(AphrontDatabaseConnection $conn_r) {
39
47
if ($this->limit && $this->offset) {
40
48
return qsprintf($conn_r, 'LIMIT %d, %d', $this->offset, $this->limit);
41
49
} else if ($this->limit) {
+2
-2
src/infrastructure/query/policy/PhabricatorCursorPagedPolicyQuery.php
+2
-2
src/infrastructure/query/policy/PhabricatorCursorPagedPolicyQuery.php
···
53
53
}
54
54
55
55
final protected function buildLimitClause(AphrontDatabaseConnection $conn_r) {
56
-
if ($this->getLimit()) {
57
-
return qsprintf($conn_r, 'LIMIT %d', $this->getLimit());
56
+
if ($this->getRawResultLimit()) {
57
+
return qsprintf($conn_r, 'LIMIT %d', $this->getRawResultLimit());
58
58
} else {
59
59
return '';
60
60
}
+191
-13
src/infrastructure/query/policy/PhabricatorPolicyQuery.php
+191
-13
src/infrastructure/query/policy/PhabricatorPolicyQuery.php
···
16
16
* limitations under the License.
17
17
*/
18
18
19
-
abstract class PhabricatorPolicyQuery extends PhabricatorQuery {
19
+
/**
20
+
* A @{class:PhabricatorQuery} which filters results according to visibility
21
+
* policies for the querying user. Broadly, this class allows you to implement
22
+
* a query that returns only objects the user is allowed to see.
23
+
*
24
+
* $results = id(new ExampleQuery())
25
+
* ->setViewer($user)
26
+
* ->withConstraint($example)
27
+
* ->execute();
28
+
*
29
+
* Normally, you should extend @{class:PhabricatorCursorPagedPolicyQuery}, not
30
+
* this class. @{class:PhabricatorCursorPagedPolicyQuery} provides a more
31
+
* practical interface for building usable queries against most object types.
32
+
*
33
+
* NOTE: Although this class extends @{class:PhabricatorOffsetPagedQuery},
34
+
* offset paging with policy filtering is not efficient. All results must be
35
+
* loaded into the application and filtered here: skipping `N` rows via offset
36
+
* is an `O(N)` operation with a large constant. Prefer cursor-based paging
37
+
* with @{class:PhabricatorCursorPagedPolicyQuery}, which can filter far more
38
+
* efficiently in MySQL.
39
+
*
40
+
* @task config Query Configuration
41
+
* @task exec Executing Queries
42
+
* @task policyimpl Policy Query Implementation
43
+
*/
44
+
abstract class PhabricatorPolicyQuery extends PhabricatorOffsetPagedQuery {
20
45
21
-
private $limit;
22
46
private $viewer;
23
47
private $raisePolicyExceptions;
48
+
private $rawResultLimit;
49
+
50
+
51
+
/* -( Query Configuration )------------------------------------------------ */
52
+
24
53
25
-
final public function setViewer($viewer) {
54
+
/**
55
+
* Set the viewer who is executing the query. Results will be filtered
56
+
* according to the viewer's capabilities. You must set a viewer to execute
57
+
* a policy query.
58
+
*
59
+
* @param PhabricatorUser The viewing user.
60
+
* @return this
61
+
* @task config
62
+
*/
63
+
final public function setViewer(PhabricatorUser $viewer) {
26
64
$this->viewer = $viewer;
27
65
return $this;
28
66
}
29
67
68
+
69
+
/**
70
+
* Get the query's viewer.
71
+
*
72
+
* @return PhabricatorUser The viewing user.
73
+
* @task config
74
+
*/
30
75
final public function getViewer() {
31
76
return $this->viewer;
32
77
}
33
78
34
-
final public function setLimit($limit) {
35
-
$this->limit = $limit;
36
-
return $this;
37
-
}
79
+
80
+
/* -( Query Execution )---------------------------------------------------- */
38
81
39
-
final public function getLimit() {
40
-
return $this->limit;
41
-
}
42
82
83
+
/**
84
+
* Execute the query, expecting a single result. This method simplifies
85
+
* loading objects for detail pages or edit views.
86
+
*
87
+
* // Load one result by ID.
88
+
* $obj = id(new ExampleQuery())
89
+
* ->setViewer($user)
90
+
* ->withIDs(array($id))
91
+
* ->executeOne();
92
+
* if (!$obj) {
93
+
* return new Aphront404Response();
94
+
* }
95
+
*
96
+
* If zero results match the query, this method returns `null`.
97
+
*
98
+
* If one result matches the query, this method returns that result.
99
+
*
100
+
* If two or more results match the query, this method throws an exception.
101
+
* You should use this method only when the query constraints guarantee at
102
+
* most one match (e.g., selecting a specific ID or PHID).
103
+
*
104
+
* If one result matches the query but it is caught by the policy filter (for
105
+
* example, the user is trying to view or edit an object which exists but
106
+
* which they do not have permission to see) a policy exception is thrown.
107
+
*
108
+
* @return mixed Single result, or null.
109
+
* @task exec
110
+
*/
43
111
final public function executeOne() {
44
112
45
113
$this->raisePolicyExceptions = true;
···
53
121
if (count($results) > 1) {
54
122
throw new Exception("Expected a single result!");
55
123
}
124
+
125
+
if (!$results) {
126
+
return null;
127
+
}
128
+
56
129
return head($results);
57
130
}
58
131
132
+
133
+
/**
134
+
* Execute the query, loading all visible results.
135
+
*
136
+
* @return list<PhabricatorPolicyInterface> Result objects.
137
+
* @task exec
138
+
*/
59
139
final public function execute() {
60
140
if (!$this->viewer) {
61
141
throw new Exception("Call setViewer() before execute()!");
···
69
149
70
150
$filter->raisePolicyExceptions($this->raisePolicyExceptions);
71
151
152
+
$offset = (int)$this->getOffset();
153
+
$limit = (int)$this->getLimit();
154
+
$count = 0;
155
+
156
+
$need = null;
157
+
if ($offset) {
158
+
$need = $offset + $limit;
159
+
}
160
+
161
+
$this->willExecute();
162
+
72
163
do {
164
+
165
+
// Figure out how many results to load. "0" means "all results".
166
+
$load = 0;
167
+
if ($need && ($count < $offset)) {
168
+
// This cap is just an arbitrary limit to keep memory usage from going
169
+
// crazy for large offsets; we can't execute them efficiently, but
170
+
// it should be possible to execute them without crashing.
171
+
$load = min($need, 1024);
172
+
} else if ($limit) {
173
+
// Otherwise, just load the number of rows we're after. Note that it
174
+
// might be more efficient to load more rows than this (if we expect
175
+
// about 5% of objects to be filtered, loading 105% of the limit might
176
+
// be better) or fewer rows than this (if we already have 95 rows and
177
+
// only need 100, loading only 5 rows might be better), but we currently
178
+
// just use the simplest heuristic since we don't have enough data
179
+
// about policy queries in the real world to tweak it.
180
+
$load = $limit;
181
+
}
182
+
$this->rawResultLimit = $load;
183
+
184
+
73
185
$page = $this->loadPage();
74
186
75
187
$visible = $filter->apply($page);
76
188
foreach ($visible as $key => $result) {
77
-
$results[$key] = $result;
78
-
if ($this->getLimit() && count($results) >= $this->getLimit()) {
189
+
++$count;
190
+
191
+
// If we have an offset, we just ignore that many results and start
192
+
// storing them only once we've hit the offset. This reduces memory
193
+
// requirements for large offsets, compared to storing them all and
194
+
// slicing them away later.
195
+
if ($count > $offset) {
196
+
$results[$key] = $result;
197
+
}
198
+
199
+
if ($need && ($count >= $need)) {
200
+
// If we have all the rows we need, break out of the paging query.
79
201
break 2;
80
202
}
81
203
}
82
204
83
-
if (!$this->getLimit() || (count($page) < $this->getLimit())) {
205
+
if (!$load) {
206
+
// If we don't have a load count, we loaded all the results. We do
207
+
// not need to load another page.
208
+
break;
209
+
}
210
+
211
+
if (count($page) < $load) {
212
+
// If we have a load count but the unfiltered results contained fewer
213
+
// objects, we know this was the last page of objects; we do not need
214
+
// to load another page because we can deduce it would be empty.
84
215
break;
85
216
}
86
217
···
90
221
return $results;
91
222
}
92
223
224
+
225
+
/* -( Policy Query Implementation )---------------------------------------- */
226
+
227
+
228
+
/**
229
+
* Get the number of results @{method:loadPage} should load. If the value is
230
+
* 0, @{method:loadPage} should load all available results.
231
+
*
232
+
* @return int The number of results to load, or 0 for all results.
233
+
* @task policyimpl
234
+
*/
235
+
final protected function getRawResultLimit() {
236
+
return $this->rawResultLimit;
237
+
}
238
+
239
+
240
+
/**
241
+
* Hook invoked before query execution. Generally, implementations should
242
+
* reset any internal cursors.
243
+
*
244
+
* @return void
245
+
* @task policyimpl
246
+
*/
247
+
protected function willExecute() {
248
+
return;
249
+
}
250
+
251
+
252
+
/**
253
+
* Load a raw page of results. Generally, implementations should load objects
254
+
* from the database. They should attempt to return the number of results
255
+
* hinted by @{method:getRawResultLimit}.
256
+
*
257
+
* @return list<PhabricatorPolicyInterface> List of filterable policy objects.
258
+
* @task policyimpl
259
+
*/
93
260
abstract protected function loadPage();
261
+
262
+
263
+
/**
264
+
* Update internal state so that the next call to @{method:loadPage} will
265
+
* return new results. Generally, you should adjust a cursor position based
266
+
* on the provided result page.
267
+
*
268
+
* @param list<PhabricatorPolicyInterface> The current page of results.
269
+
* @return void
270
+
* @task policyimpl
271
+
*/
94
272
abstract protected function nextPage(array $page);
95
273
96
274
}