@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Reduce badness in viewing large files in the Diffusion web UI
Summary:
Fixes T8597. Second issue there is that if you look at a huge file in Diffusion (like `/path/to/300MB.pdf`) we pull the whole thing over Conduit upfront, then try to shove it into file storage.
Instead, pull no more than the chunk limit (normally 4MB) and don't spend more than 10s pulling data.
If we get 4MB of data and/or time out, just fail with a message in the vein of "this is a really big file".
Eventually, we could improve this:
- We can determine the //size// of very large files correctly in at least some VCSes, this just takes a little more work. This would let us show the true filesize, at least.
- We could eventually stream the data out of the VCS, but we can't stream data over Conduit right now and this is a lot of work.
This is just "stop crashing".
Test Plan: Changed limits to 0.01 seconds and 8 bytes and saw reasonable errors. Changed them back and got normal beahvior.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: epriestley
Maniphest Tasks: T8597
Differential Revision: https://secure.phabricator.com/D13348
+101
-9
+16
src/applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php
+16
src/applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php
···
20
20
'path' => 'required string',
21
21
'commit' => 'required string',
22
22
'needsBlame' => 'optional bool',
23
+
'timeout' => 'optional int',
24
+
'byteLimit' => 'optional int',
23
25
);
24
26
}
25
27
···
31
33
$file_query
32
34
->setViewer($request->getUser())
33
35
->setNeedsBlame($needs_blame);
36
+
37
+
$timeout = $request->getValue('timeout');
38
+
if ($timeout) {
39
+
$file_query->setTimeout($timeout);
40
+
}
41
+
42
+
$byte_limit = $request->getValue('byteLimit');
43
+
if ($byte_limit) {
44
+
$file_query->setByteLimit($byte_limit);
45
+
}
46
+
34
47
$file_content = $file_query->loadFileContent();
48
+
35
49
if ($needs_blame) {
36
50
list($text_list, $rev_list, $blame_dict) = $file_query->getBlameData();
37
51
} else {
38
52
$text_list = $rev_list = $blame_dict = array();
39
53
}
54
+
40
55
$file_content
41
56
->setBlameDict($blame_dict)
42
57
->setRevList($rev_list)
43
58
->setTextList($text_list);
59
+
44
60
return $file_content->toDictionary();
45
61
}
46
62
+40
-8
src/applications/diffusion/controller/DiffusionBrowseFileController.php
+40
-8
src/applications/diffusion/controller/DiffusionBrowseFileController.php
···
54
54
$needs_blame = ($show_blame && !$show_color) ||
55
55
($show_blame && $request->isAjax());
56
56
57
+
$params = array(
58
+
'commit' => $drequest->getCommit(),
59
+
'path' => $drequest->getPath(),
60
+
'needsBlame' => $needs_blame,
61
+
);
62
+
63
+
$byte_limit = null;
64
+
if ($view !== 'raw') {
65
+
$byte_limit = PhabricatorFileStorageEngine::getChunkThreshold();
66
+
$time_limit = 10;
67
+
68
+
$params += array(
69
+
'timeout' => $time_limit,
70
+
'byteLimit' => $byte_limit,
71
+
);
72
+
}
73
+
57
74
$file_content = DiffusionFileContent::newFromConduit(
58
75
$this->callConduitWithDiffusionRequest(
59
76
'diffusion.filecontentquery',
60
-
array(
61
-
'commit' => $drequest->getCommit(),
62
-
'path' => $drequest->getPath(),
63
-
'needsBlame' => $needs_blame,
64
-
)));
77
+
$params));
65
78
$data = $file_content->getCorpus();
66
79
67
80
if ($view === 'raw') {
···
71
84
$this->loadLintMessages();
72
85
$this->coverage = $drequest->loadCoverage();
73
86
74
-
$binary_uri = null;
75
-
if (ArcanistDiffUtils::isHeuristicBinaryFile($data)) {
87
+
if ($byte_limit && (strlen($data) == $byte_limit)) {
88
+
$corpus = $this->buildErrorCorpus(
89
+
pht(
90
+
'This file is larger than %s byte(s), and too large to display '.
91
+
'in the web UI.',
92
+
$byte_limit));
93
+
} else if (ArcanistDiffUtils::isHeuristicBinaryFile($data)) {
76
94
$file = $this->loadFileForData($path, $data);
77
95
$file_uri = $file->getBestURI();
78
96
···
80
98
$corpus = $this->buildImageCorpus($file_uri);
81
99
} else {
82
100
$corpus = $this->buildBinaryCorpus($file_uri, $data);
83
-
$binary_uri = $file_uri;
84
101
}
85
102
} else {
86
103
// Build the content of the file.
···
932
949
$header = id(new PHUIHeaderView())
933
950
->setHeader(pht('Details'))
934
951
->addActionLink($file);
952
+
953
+
$box = id(new PHUIObjectBoxView())
954
+
->setHeader($header)
955
+
->appendChild($text);
956
+
957
+
return $box;
958
+
}
959
+
960
+
private function buildErrorCorpus($message) {
961
+
$text = id(new PHUIBoxView())
962
+
->addPadding(PHUI::PADDING_LARGE)
963
+
->appendChild($message);
964
+
965
+
$header = id(new PHUIHeaderView())
966
+
->setHeader(pht('Details'));
935
967
936
968
$box = id(new PHUIObjectBoxView())
937
969
->setHeader($header)
+45
-1
src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php
+45
-1
src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php
···
11
11
private $needsBlame;
12
12
private $fileContent;
13
13
private $viewer;
14
+
private $timeout;
15
+
private $byteLimit;
16
+
17
+
public function setTimeout($timeout) {
18
+
$this->timeout = $timeout;
19
+
return $this;
20
+
}
21
+
22
+
public function getTimeout() {
23
+
return $this->timeout;
24
+
}
25
+
26
+
public function setByteLimit($byte_limit) {
27
+
$this->byteLimit = $byte_limit;
28
+
return $this;
29
+
}
30
+
31
+
public function getByteLimit() {
32
+
return $this->byteLimit;
33
+
}
14
34
15
35
final public static function newFromDiffusionRequest(
16
36
DiffusionRequest $request) {
···
21
41
abstract protected function executeQueryFromFuture(Future $future);
22
42
23
43
final public function loadFileContentFromFuture(Future $future) {
24
-
$this->fileContent = $this->executeQueryFromFuture($future);
44
+
45
+
if ($this->timeout) {
46
+
$future->setTimeout($this->timeout);
47
+
}
48
+
49
+
if ($this->getByteLimit()) {
50
+
$future->setStdoutSizeLimit($this->getByteLimit());
51
+
}
52
+
53
+
try {
54
+
$file_content = $this->executeQueryFromFuture($future);
55
+
} catch (CommandException $ex) {
56
+
if (!$future->getWasKilledByTimeout()) {
57
+
throw $ex;
58
+
}
59
+
60
+
$message = pht(
61
+
'<Attempt to load this file was terminated after %s second(s).>',
62
+
$this->timeout);
63
+
64
+
$file_content = new DiffusionFileContent();
65
+
$file_content->setCorpus($message);
66
+
}
67
+
68
+
$this->fileContent = $file_content;
25
69
26
70
$repository = $this->getRequest()->getRepository();
27
71
$try_encoding = $repository->getDetail('encoding');