@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Fix some file policy issues and add a "Query Workspace"
Summary:
Ref T603. Several issues here:
1. Currently, `FileQuery` does not actually respect object attachment edges when doing policy checks. Everything else works fine, but this was missing an `array_keys()`.
2. Once that's fixed, we hit a bunch of recursion issues. For example, when loading a User we load the profile picture, and then that loads the User, and that loads the profile picture, etc.
3. Introduce a "Query Workspace", which holds objects we know we've loaded and know we can see but haven't finished filtering and/or attaching data to. This allows subqueries to look up objects instead of querying for them.
- We can probably generalize this a bit to make a few other queries more efficient. Pholio currently has a similar (but less general) "mock cache". However, it's keyed by ID instead of PHID so it's not easy to reuse this right now.
This is a bit complex for the problem being solved, but I think it's the cleanest approach and I believe the primitive will be useful in the future.
Test Plan: Looked at pastes, macros, mocks and projects as a logged-in and logged-out user.
Reviewers: btrahan
Reviewed By: btrahan
CC: aran
Maniphest Tasks: T603
Differential Revision: https://secure.phabricator.com/D7309
+236
-60
+2
src/applications/files/query/PhabricatorFileQuery.php
+2
src/applications/files/query/PhabricatorFileQuery.php
···
128
128
$object_phids[$phid] = true;
129
129
}
130
130
}
131
+
$object_phids = array_keys($object_phids);
131
132
132
133
// Now, load the objects.
133
134
134
135
$objects = array();
135
136
if ($object_phids) {
136
137
$objects = id(new PhabricatorObjectQuery())
138
+
->setParentQuery($this)
137
139
->setViewer($this->getViewer())
138
140
->withPHIDs($object_phids)
139
141
->execute();
+3
-1
src/applications/files/storage/PhabricatorFile.php
+3
-1
src/applications/files/storage/PhabricatorFile.php
+2
-1
src/applications/macro/query/PhabricatorMacroQuery.php
+2
-1
src/applications/macro/query/PhabricatorMacroQuery.php
···
191
191
return $this->formatWhereClause($where);
192
192
}
193
193
194
-
protected function willFilterPage(array $macros) {
194
+
protected function didFilterPage(array $macros) {
195
195
$file_phids = mpull($macros, 'getFilePHID');
196
196
$files = id(new PhabricatorFileQuery())
197
197
->setViewer($this->getViewer())
198
+
->setParentQuery($this)
198
199
->withPHIDs($file_phids)
199
200
->execute();
200
201
$files = mpull($files, null, 'getPHID');
+2
-1
src/applications/paste/query/PhabricatorPasteQuery.php
+2
-1
src/applications/paste/query/PhabricatorPasteQuery.php
···
87
87
return $pastes;
88
88
}
89
89
90
-
protected function willFilterPage(array $pastes) {
90
+
protected function didFilterPage(array $pastes) {
91
91
if ($this->needRawContent) {
92
92
$pastes = $this->loadRawContent($pastes);
93
93
}
···
163
163
private function loadRawContent(array $pastes) {
164
164
$file_phids = mpull($pastes, 'getFilePHID');
165
165
$files = id(new PhabricatorFileQuery())
166
+
->setParentQuery($this)
166
167
->setViewer($this->getViewer())
167
168
->withPHIDs($file_phids)
168
169
->execute();
+4
-1
src/applications/people/query/PhabricatorPeopleQuery.php
+4
-1
src/applications/people/query/PhabricatorPeopleQuery.php
···
113
113
$table->putInSet(new LiskDAOSet());
114
114
}
115
115
116
-
$users = $table->loadAllFromArray($data);
116
+
return $table->loadAllFromArray($data);
117
+
}
117
118
119
+
protected function didFilterPage(array $users) {
118
120
if ($this->needProfile) {
119
121
$user_list = mpull($users, null, 'getPHID');
120
122
$profiles = new PhabricatorUserProfile();
···
138
140
$user_profile_file_phids = array_filter($user_profile_file_phids);
139
141
if ($user_profile_file_phids) {
140
142
$files = id(new PhabricatorFileQuery())
143
+
->setParentQuery($this)
141
144
->setViewer($this->getViewer())
142
145
->withPHIDs($user_profile_file_phids)
143
146
->execute();
+15
-1
src/applications/phid/query/PhabricatorObjectQuery.php
+15
-1
src/applications/phid/query/PhabricatorObjectQuery.php
···
104
104
}
105
105
106
106
private function loadObjectsByPHID(array $types, array $phids) {
107
+
$results = array();
108
+
109
+
$workspace = $this->getObjectsFromWorkspace($phids);
110
+
111
+
foreach ($phids as $key => $phid) {
112
+
if (isset($workspace[$phid])) {
113
+
$results[$phid] = $workspace[$phid];
114
+
unset($phids[$key]);
115
+
}
116
+
}
117
+
118
+
if (!$phids) {
119
+
return $results;
120
+
}
121
+
107
122
$groups = array();
108
123
foreach ($phids as $phid) {
109
124
$type = phid_get_type($phid);
110
125
$groups[$type][] = $phid;
111
126
}
112
127
113
-
$results = array();
114
128
foreach ($groups as $type => $group) {
115
129
if (isset($types[$type])) {
116
130
$objects = $types[$type]->loadObjects($this, $group);
+28
-21
src/applications/pholio/query/PholioImageQuery.php
+28
-21
src/applications/pholio/query/PholioImageQuery.php
···
103
103
protected function willFilterPage(array $images) {
104
104
assert_instances_of($images, 'PholioImage');
105
105
106
+
if ($this->getMockCache()) {
107
+
$mocks = $this->getMockCache();
108
+
} else {
109
+
$mock_ids = mpull($images, 'getMockID');
110
+
// DO NOT set needImages to true; recursion results!
111
+
$mocks = id(new PholioMockQuery())
112
+
->setViewer($this->getViewer())
113
+
->withIDs($mock_ids)
114
+
->execute();
115
+
$mocks = mpull($mocks, null, 'getID');
116
+
}
117
+
foreach ($images as $index => $image) {
118
+
$mock = idx($mocks, $image->getMockID());
119
+
if ($mock) {
120
+
$image->attachMock($mock);
121
+
} else {
122
+
// mock is missing or we can't see it
123
+
unset($images[$index]);
124
+
}
125
+
}
126
+
127
+
return $images;
128
+
}
129
+
130
+
protected function didFilterPage(array $images) {
131
+
assert_instances_of($images, 'PholioImage');
132
+
106
133
$file_phids = mpull($images, 'getFilePHID');
107
134
108
135
$all_files = id(new PhabricatorFileQuery())
136
+
->setParentQuery($this)
109
137
->setViewer($this->getViewer())
110
138
->withPHIDs($file_phids)
111
139
->execute();
···
127
155
if ($this->needInlineComments) {
128
156
$inlines = idx($all_inline_comments, $image->getID(), array());
129
157
$image->attachInlineComments($inlines);
130
-
}
131
-
}
132
-
133
-
if ($this->getMockCache()) {
134
-
$mocks = $this->getMockCache();
135
-
} else {
136
-
$mock_ids = mpull($images, 'getMockID');
137
-
// DO NOT set needImages to true; recursion results!
138
-
$mocks = id(new PholioMockQuery())
139
-
->setViewer($this->getViewer())
140
-
->withIDs($mock_ids)
141
-
->execute();
142
-
$mocks = mpull($mocks, null, 'getID');
143
-
}
144
-
foreach ($images as $index => $image) {
145
-
$mock = idx($mocks, $image->getMockID());
146
-
if ($mock) {
147
-
$image->attachMock($mock);
148
-
} else {
149
-
// mock is missing or we can't see it
150
-
unset($images[$index]);
151
158
}
152
159
}
153
160
+4
src/applications/project/controller/PhabricatorProjectProfileController.php
+4
src/applications/project/controller/PhabricatorProjectProfileController.php
+38
-33
src/applications/project/query/PhabricatorProjectQuery.php
+38
-33
src/applications/project/query/PhabricatorProjectQuery.php
···
114
114
($row['viewerIsMember'] !== null));
115
115
}
116
116
}
117
+
}
117
118
118
-
if ($this->needProfiles) {
119
-
$profiles = id(new PhabricatorProjectProfile())->loadAllWhere(
120
-
'projectPHID IN (%Ls)',
121
-
mpull($projects, 'getPHID'));
122
-
$profiles = mpull($profiles, null, 'getProjectPHID');
119
+
return $projects;
120
+
}
123
121
124
-
$default = null;
122
+
protected function didFilterPage(array $projects) {
123
+
if ($this->needProfiles) {
124
+
$profiles = id(new PhabricatorProjectProfile())->loadAllWhere(
125
+
'projectPHID IN (%Ls)',
126
+
mpull($projects, 'getPHID'));
127
+
$profiles = mpull($profiles, null, 'getProjectPHID');
125
128
126
-
if ($profiles) {
127
-
$file_phids = mpull($profiles, 'getProfileImagePHID');
128
-
$files = id(new PhabricatorFileQuery())
129
-
->setViewer($this->getViewer())
130
-
->withPHIDs($file_phids)
131
-
->execute();
132
-
$files = mpull($files, null, 'getPHID');
133
-
foreach ($profiles as $profile) {
134
-
$file = idx($files, $profile->getProfileImagePHID());
135
-
if (!$file) {
136
-
if (!$default) {
137
-
$default = PhabricatorFile::loadBuiltin(
138
-
$this->getViewer(),
139
-
'profile.png');
140
-
}
141
-
$file = $default;
142
-
}
143
-
$profile->attachProfileImageFile($file);
144
-
}
145
-
}
129
+
$default = null;
146
130
147
-
foreach ($projects as $project) {
148
-
$profile = idx($profiles, $project->getPHID());
149
-
if (!$profile) {
131
+
if ($profiles) {
132
+
$file_phids = mpull($profiles, 'getProfileImagePHID');
133
+
$files = id(new PhabricatorFileQuery())
134
+
->setParentQuery($this)
135
+
->setViewer($this->getViewer())
136
+
->withPHIDs($file_phids)
137
+
->execute();
138
+
$files = mpull($files, null, 'getPHID');
139
+
foreach ($profiles as $profile) {
140
+
$file = idx($files, $profile->getProfileImagePHID());
141
+
if (!$file) {
150
142
if (!$default) {
151
143
$default = PhabricatorFile::loadBuiltin(
152
144
$this->getViewer(),
153
145
'profile.png');
154
146
}
155
-
$profile = id(new PhabricatorProjectProfile())
156
-
->setProjectPHID($project->getPHID())
157
-
->attachProfileImageFile($default);
147
+
$file = $default;
148
+
}
149
+
$profile->attachProfileImageFile($file);
150
+
}
151
+
}
152
+
153
+
foreach ($projects as $project) {
154
+
$profile = idx($profiles, $project->getPHID());
155
+
if (!$profile) {
156
+
if (!$default) {
157
+
$default = PhabricatorFile::loadBuiltin(
158
+
$this->getViewer(),
159
+
'profile.png');
158
160
}
159
-
$project->attachProfile($profile);
161
+
$profile = id(new PhabricatorProjectProfile())
162
+
->setProjectPHID($project->getPHID())
163
+
->attachProfileImageFile($default);
160
164
}
165
+
$project->attachProfile($profile);
161
166
}
162
167
}
163
168
+138
-1
src/infrastructure/query/policy/PhabricatorPolicyAwareQuery.php
+138
-1
src/infrastructure/query/policy/PhabricatorPolicyAwareQuery.php
···
33
33
private $parentQuery;
34
34
private $rawResultLimit;
35
35
private $capabilities;
36
+
private $workspace = array();
36
37
37
38
38
39
/* -( Query Configuration )------------------------------------------------ */
···
229
230
$visible = $filter->apply($maybe_visible);
230
231
}
231
232
233
+
if ($visible) {
234
+
$this->putObjectsInWorkspace($this->getWorkspaceMapForPage($visible));
235
+
$visible = $this->didFilterPage($visible);
236
+
}
237
+
232
238
$removed = array();
233
239
foreach ($maybe_visible as $key => $object) {
234
240
if (empty($visible[$key])) {
···
300
306
}
301
307
302
308
309
+
/* -( Query Workspace )---------------------------------------------------- */
310
+
311
+
312
+
/**
313
+
* Put a map of objects into the query workspace. Many queries perform
314
+
* subqueries, which can eventually end up loading the same objects more than
315
+
* once (often to perform policy checks).
316
+
*
317
+
* For example, loading a user may load the user's profile image, which might
318
+
* load the user object again in order to verify that the viewer has
319
+
* permission to see the file.
320
+
*
321
+
* The "query workspace" allows queries to load objects from elsewhere in a
322
+
* query block instead of refetching them.
323
+
*
324
+
* When using the query workspace, it's important to obey two rules:
325
+
*
326
+
* **Never put objects into the workspace which the viewer may not be able
327
+
* to see**. You need to apply all policy filtering //before// putting
328
+
* objects in the workspace. Otherwise, subqueries may read the objects and
329
+
* use them to permit access to content the user shouldn't be able to view.
330
+
*
331
+
* **Fully enrich objects pulled from the workspace.** After pulling objects
332
+
* from the workspace, you still need to load and attach any additional
333
+
* content the query requests. Otherwise, a query might return objects without
334
+
* requested content.
335
+
*
336
+
* Generally, you do not need to update the workspace yourself: it is
337
+
* automatically populated as a side effect of objects surviving policy
338
+
* filtering.
339
+
*
340
+
* @param map<phid, PhabricatorPolicyInterface> Objects to add to the query
341
+
* workspace.
342
+
* @return this
343
+
* @task workspace
344
+
*/
345
+
public function putObjectsInWorkspace(array $objects) {
346
+
assert_instances_of($objects, 'PhabricatorPolicyInterface');
347
+
348
+
$viewer_phid = $this->getViewer()->getPHID();
349
+
350
+
// The workspace is scoped per viewer to prevent accidental contamination.
351
+
if (empty($this->workspace[$viewer_phid])) {
352
+
$this->workspace[$viewer_phid] = array();
353
+
}
354
+
355
+
$this->workspace[$viewer_phid] += $objects;
356
+
357
+
return $this;
358
+
}
359
+
360
+
361
+
/**
362
+
* Retrieve objects from the query workspace. For more discussion about the
363
+
* workspace mechanism, see @{method:putObjectsInWorkspace}. This method
364
+
* searches both the current query's workspace and the workspaces of parent
365
+
* queries.
366
+
*
367
+
* @param list<phid> List of PHIDs to retreive.
368
+
* @return this
369
+
* @task workspace
370
+
*/
371
+
public function getObjectsFromWorkspace(array $phids) {
372
+
$viewer_phid = $this->getViewer()->getPHID();
373
+
374
+
$results = array();
375
+
foreach ($phids as $key => $phid) {
376
+
if (isset($this->workspace[$viewer_phid][$phid])) {
377
+
$results[$phid] = $this->workspace[$viewer_phid][$phid];
378
+
unset($phids[$key]);
379
+
}
380
+
}
381
+
382
+
if ($phids && $this->getParentQuery()) {
383
+
$results += $this->getParentQuery()->getObjectsFromWorkspace($phids);
384
+
}
385
+
386
+
return $results;
387
+
}
388
+
389
+
390
+
/**
391
+
* Convert a result page to a `<phid, PhabricatorPolicyInterface>` map.
392
+
*
393
+
* @param list<PhabricatorPolicyInterface> Objects.
394
+
* @return map<phid, PhabricatorPolicyInterface> Map of objects which can
395
+
* be put into the workspace.
396
+
* @task workspace
397
+
*/
398
+
protected function getWorkspaceMapForPage(array $results) {
399
+
$map = array();
400
+
foreach ($results as $result) {
401
+
$phid = $result->getPHID();
402
+
if ($phid !== null) {
403
+
$map[$phid] = $result;
404
+
}
405
+
}
406
+
407
+
return $map;
408
+
}
409
+
410
+
303
411
/* -( Policy Query Implementation )---------------------------------------- */
304
412
305
413
···
353
461
/**
354
462
* Hook for applying a page filter prior to the privacy filter. This allows
355
463
* you to drop some items from the result set without creating problems with
356
-
* pagination or cursor updates.
464
+
* pagination or cursor updates. You can also load and attach data which is
465
+
* required to perform policy filtering.
466
+
*
467
+
* Generally, you should load non-policy data and perform non-policy filtering
468
+
* later, in @{method:didFilterPage}. Strictly fewer objects will make it that
469
+
* far (so the program will load less data) and subqueries from that context
470
+
* can use the query workspace to further reduce query load.
357
471
*
358
472
* This method will only be called if data is available. Implementations
359
473
* do not need to handle the case of no results specially.
···
363
477
* @task policyimpl
364
478
*/
365
479
protected function willFilterPage(array $page) {
480
+
return $page;
481
+
}
482
+
483
+
/**
484
+
* Hook for performing additional non-policy loading or filtering after an
485
+
* object has satisfied all policy checks. Generally, this means loading and
486
+
* attaching related data.
487
+
*
488
+
* Subqueries executed during this phase can use the query workspace, which
489
+
* may improve performance or make circular policies resolvable. Data which
490
+
* is not necessary for policy filtering should generally be loaded here.
491
+
*
492
+
* This callback can still filter objects (for example, if attachable data
493
+
* is discovered to not exist), but should not do so for policy reasons.
494
+
*
495
+
* This method will only be called if data is available. Implementations do
496
+
* not need to handle the case of no results specially.
497
+
*
498
+
* @param list<wild> Results from @{method:willFilterPage()}.
499
+
* @return list<PhabricatorPolicyInterface> Objects after additional
500
+
* non-policy processing.
501
+
*/
502
+
protected function didFilterPage(array $page) {
366
503
return $page;
367
504
}
368
505