@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Improve Diffusion behavior for directories with impressive numbers of files
Summary:
Fixes T4366. Two years ago, Facebook put 16,000 files in a directory. Today, the page has nearly loaded.
Paginate large directories.
Test Plan:
- Viewed home and browse views in Git, Mercurial and Subversion.
I put an artificially small page size (5) on home:
{F1055653}
I pushed 16,000 files to a directory and paged through them. Here's the last page, which rendered in about 200ms:
{F1055655}
Our behavior is a bit better than GitHub here, which shows only the first 1,000 files, disables pagination, and can't retrieve history for the files:
{F1055656}
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T4366
Differential Revision: https://secure.phabricator.com/D14956
+87
-13
+39
-5
src/applications/diffusion/conduit/DiffusionBrowseQueryConduitAPIMethod.php
+39
-5
src/applications/diffusion/conduit/DiffusionBrowseQueryConduitAPIMethod.php
···
22
22
'path' => 'optional string',
23
23
'commit' => 'optional string',
24
24
'needValidityOnly' => 'optional bool',
25
+
'limit' => 'optional int',
26
+
'offset' => 'optional int',
25
27
);
26
28
}
27
29
···
35
37
$repository = $drequest->getRepository();
36
38
$path = $request->getValue('path');
37
39
$commit = $request->getValue('commit');
40
+
$offset = (int)$request->getValue('offset');
41
+
$limit = (int)$request->getValue('limit');
38
42
$result = $this->getEmptyResultSet();
39
43
40
44
if ($path == '') {
···
99
103
$prefix = '';
100
104
}
101
105
106
+
$count = 0;
102
107
$results = array();
103
108
foreach (explode("\0", rtrim($stdout)) as $line) {
104
109
// NOTE: Limit to 5 components so we parse filenames with spaces in them
···
140
145
$path_result->setFileType($file_type);
141
146
$path_result->setFileSize($size);
142
147
143
-
$results[] = $path_result;
148
+
if ($count >= $offset) {
149
+
$results[] = $path_result;
150
+
}
151
+
152
+
$count++;
153
+
154
+
if ($limit && $count >= ($offset + $limit)) {
155
+
break;
156
+
}
144
157
}
145
158
146
159
// If we identified submodules, lookup the module info at this commit to
···
196
209
$repository = $drequest->getRepository();
197
210
$path = $request->getValue('path');
198
211
$commit = $request->getValue('commit');
212
+
$offset = (int)$request->getValue('offset');
213
+
$limit = (int)$request->getValue('limit');
199
214
$result = $this->getEmptyResultSet();
200
215
201
216
···
215
230
// but ours do.
216
231
$trim_len = $match_len ? $match_len + 1 : 0;
217
232
233
+
$count = 0;
218
234
foreach ($entire_manifest as $path) {
219
235
if (strncmp($path, $match_against, $match_len)) {
220
236
continue;
···
236
252
} else {
237
253
$type = DifferentialChangeType::FILE_DIRECTORY;
238
254
}
239
-
$results[reset($parts)] = $type;
255
+
256
+
if ($count >= $offset) {
257
+
$results[reset($parts)] = $type;
258
+
}
259
+
260
+
$count++;
261
+
262
+
if ($limit && ($count >= ($offset + $limit))) {
263
+
break;
264
+
}
240
265
}
241
266
242
267
foreach ($results as $key => $type) {
···
266
291
$repository = $drequest->getRepository();
267
292
$path = $request->getValue('path');
268
293
$commit = $request->getValue('commit');
294
+
$offset = (int)$request->getValue('offset');
295
+
$limit = (int)$request->getValue('limit');
269
296
$result = $this->getEmptyResultSet();
270
297
271
298
$subpath = $repository->getDetail('svn-subpath');
···
422
449
$path_normal = DiffusionPathIDQuery::normalizePath($path);
423
450
424
451
$results = array();
452
+
$count = 0;
425
453
foreach ($browse as $file) {
426
454
427
455
$full_path = $file['pathName'];
···
431
459
$result_path = new DiffusionRepositoryPath();
432
460
$result_path->setPath($file_path);
433
461
$result_path->setFullPath($full_path);
434
-
// $result_path->setHash($hash);
435
462
$result_path->setFileType($file['fileType']);
436
-
// $result_path->setFileSize($size);
437
463
438
464
if (!empty($file['hasCommit'])) {
439
465
$commit = idx($commits, $file['svnCommit']);
···
444
470
}
445
471
}
446
472
447
-
$results[] = $result_path;
473
+
if ($count >= $offset) {
474
+
$results[] = $result_path;
475
+
}
476
+
477
+
$count++;
478
+
479
+
if ($limit && ($count >= ($offset + $limit))) {
480
+
break;
481
+
}
448
482
}
449
483
450
484
if (empty($results)) {
+22
-3
src/applications/diffusion/controller/DiffusionBrowseController.php
+22
-3
src/applications/diffusion/controller/DiffusionBrowseController.php
···
27
27
return $this->browseSearch();
28
28
}
29
29
30
+
$pager = id(new PHUIPagerView())
31
+
->readFromRequest($request);
32
+
30
33
$results = DiffusionBrowseResultSet::newFromConduit(
31
34
$this->callConduitWithDiffusionRequest(
32
35
'diffusion.browsequery',
33
36
array(
34
37
'path' => $drequest->getPath(),
35
38
'commit' => $drequest->getStableCommit(),
39
+
'offset' => $pager->getOffset(),
40
+
'limit' => $pager->getPageSize() + 1,
36
41
)));
42
+
37
43
$reason = $results->getReasonForEmptyResultSet();
38
44
$is_file = ($reason == DiffusionBrowseResultSet::REASON_IS_FILE);
39
45
40
46
if ($is_file) {
41
47
return $this->browseFile($results);
42
48
} else {
43
-
return $this->browseDirectory($results);
49
+
$paths = $results->getPaths();
50
+
$paths = $pager->sliceResults($paths);
51
+
$results->setPaths($paths);
52
+
53
+
return $this->browseDirectory($results, $pager);
44
54
}
45
55
}
46
56
···
262
272
->appendChild($content);
263
273
}
264
274
265
-
public function browseDirectory(DiffusionBrowseResultSet $results) {
275
+
public function browseDirectory(
276
+
DiffusionBrowseResultSet $results,
277
+
PHUIPagerView $pager) {
278
+
266
279
$request = $this->getRequest();
267
280
$drequest = $this->getDiffusionRequest();
268
281
$repository = $drequest->getRepository();
···
339
352
'view' => 'browse',
340
353
));
341
354
355
+
$pager_box = $this->renderTablePagerBox($pager);
356
+
342
357
return $this->newPage()
343
358
->setTitle(
344
359
array(
···
346
361
$repository->getDisplayName(),
347
362
))
348
363
->setCrumbs($crumbs)
349
-
->appendChild($content);
364
+
->appendChild(
365
+
array(
366
+
$content,
367
+
$pager_box,
368
+
));
350
369
}
351
370
352
371
private function renderSearchResults() {
+1
-1
src/applications/diffusion/controller/DiffusionController.php
+1
-1
src/applications/diffusion/controller/DiffusionController.php
···
110
110
$crumb_list = array();
111
111
112
112
// On the home page, we don't have a DiffusionRequest.
113
-
if ($this->diffusionRequest) {
113
+
if ($this->hasDiffusionRequest()) {
114
114
$drequest = $this->getDiffusionRequest();
115
115
$repository = $drequest->getRepository();
116
116
} else {
+23
-3
src/applications/diffusion/controller/DiffusionRepositoryController.php
+23
-3
src/applications/diffusion/controller/DiffusionRepositoryController.php
···
88
88
89
89
90
90
private function buildNormalContent(DiffusionRequest $drequest) {
91
+
$request = $this->getRequest();
91
92
$repository = $drequest->getRepository();
92
93
93
94
$phids = array();
···
123
124
$history_exception = $ex;
124
125
}
125
126
127
+
$browse_pager = id(new PHUIPagerView())
128
+
->readFromRequest($request);
129
+
126
130
try {
127
131
$browse_results = DiffusionBrowseResultSet::newFromConduit(
128
132
$this->callConduitWithDiffusionRequest(
···
130
134
array(
131
135
'path' => $drequest->getPath(),
132
136
'commit' => $drequest->getCommit(),
137
+
'limit' => $browse_pager->getPageSize() + 1,
133
138
)));
134
139
$browse_paths = $browse_results->getPaths();
140
+
$browse_paths = $browse_pager->sliceResults($browse_paths);
135
141
136
142
foreach ($browse_paths as $item) {
137
143
$data = $item->getLastCommitData();
···
178
184
$browse_results,
179
185
$browse_paths,
180
186
$browse_exception,
181
-
$handles);
187
+
$handles,
188
+
$browse_pager);
182
189
183
190
$content[] = $this->buildHistoryTable(
184
191
$history_results,
···
588
595
$browse_results,
589
596
$browse_paths,
590
597
$browse_exception,
591
-
array $handles) {
598
+
array $handles,
599
+
PHUIPagerView $pager) {
592
600
593
601
require_celerity_resource('diffusion-icons-css');
594
602
···
669
677
670
678
$browse_panel->setTable($browse_table);
671
679
672
-
return array($locate_panel, $browse_panel);
680
+
$pager->setURI($browse_uri, 'offset');
681
+
682
+
if ($pager->willShowPagingControls()) {
683
+
$pager_box = $this->renderTablePagerBox($pager);
684
+
} else {
685
+
$pager_box = null;
686
+
}
687
+
688
+
return array(
689
+
$locate_panel,
690
+
$browse_panel,
691
+
$pager_box,
692
+
);
673
693
}
674
694
675
695
private function renderCloneCommand(
+2
-1
src/applications/repository/storage/PhabricatorRepository.php
+2
-1
src/applications/repository/storage/PhabricatorRepository.php