@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Allow diffusion.filecontentquery to load data for arbitrarily large files
Summary:
Fixes T10186. After D14970, `diffusion.filecontentquery` puts the content in a file and returns the file PHID.
However, it does this in a way that doesn't go through the chunking engine, so it will fail for files larger than the chunk threshold (generally, 8MB).
Instead, stream the file from the underlying command directly into chunked storage.
Test Plan:
- Made a commit including a really big file: https://github.com/epriestley/poems/commit/4dcd4c492b31d82f0fec6f447cf3719b9cc079e0
- Used `diffusion.filecontentquery` to load file content.
- Parsed/imported commit locally.
- Used `diffusion.filecontentquery` to load content for smaller files (README, etc).
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T10186
Differential Revision: https://secure.phabricator.com/D15072
+309
-43
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
2223
2223
'PhabricatorEventListener' => 'infrastructure/events/PhabricatorEventListener.php',
2224
2224
'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php',
2225
2225
'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php',
2226
+
'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php',
2226
2227
'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php',
2227
2228
'PhabricatorExtendingPhabricatorConfigOptions' => 'applications/config/option/PhabricatorExtendingPhabricatorConfigOptions.php',
2228
2229
'PhabricatorExtensionsSetupCheck' => 'applications/config/check/PhabricatorExtensionsSetupCheck.php',
···
2312
2313
'PhabricatorFileUploadController' => 'applications/files/controller/PhabricatorFileUploadController.php',
2313
2314
'PhabricatorFileUploadDialogController' => 'applications/files/controller/PhabricatorFileUploadDialogController.php',
2314
2315
'PhabricatorFileUploadException' => 'applications/files/exception/PhabricatorFileUploadException.php',
2316
+
'PhabricatorFileUploadSource' => 'applications/files/uploadsource/PhabricatorFileUploadSource.php',
2315
2317
'PhabricatorFileinfoSetupCheck' => 'applications/config/check/PhabricatorFileinfoSetupCheck.php',
2316
2318
'PhabricatorFilesApplication' => 'applications/files/application/PhabricatorFilesApplication.php',
2317
2319
'PhabricatorFilesApplicationStorageEnginePanel' => 'applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php',
···
6504
6506
'PhabricatorEventListener' => 'PhutilEventListener',
6505
6507
'PhabricatorEventType' => 'PhutilEventType',
6506
6508
'PhabricatorExampleEventListener' => 'PhabricatorEventListener',
6509
+
'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource',
6507
6510
'PhabricatorExtendingPhabricatorConfigOptions' => 'PhabricatorApplicationConfigOptions',
6508
6511
'PhabricatorExtensionsSetupCheck' => 'PhabricatorSetupCheck',
6509
6512
'PhabricatorExternalAccount' => array(
···
6621
6624
'PhabricatorFileUploadController' => 'PhabricatorFileController',
6622
6625
'PhabricatorFileUploadDialogController' => 'PhabricatorFileController',
6623
6626
'PhabricatorFileUploadException' => 'Exception',
6627
+
'PhabricatorFileUploadSource' => 'Phobject',
6624
6628
'PhabricatorFileinfoSetupCheck' => 'PhabricatorSetupCheck',
6625
6629
'PhabricatorFilesApplication' => 'PhabricatorApplication',
6626
6630
'PhabricatorFilesApplicationStorageEnginePanel' => 'PhabricatorApplicationConfigurationPanel',
+8
-24
src/applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php
+8
-24
src/applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php
···
39
39
$file_query->setByteLimit($byte_limit);
40
40
}
41
41
42
-
$content = $file_query->execute();
42
+
$file = $file_query->execute();
43
43
44
44
$too_slow = (bool)$file_query->getExceededTimeLimit();
45
45
$too_huge = (bool)$file_query->getExceededByteLimit();
46
46
47
47
$file_phid = null;
48
48
if (!$too_slow && !$too_huge) {
49
-
$file = $this->newFile($drequest, $content);
49
+
$repository = $drequest->getRepository();
50
+
$repository_phid = $repository->getPHID();
51
+
52
+
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
53
+
$file->attachToObject($repository_phid);
54
+
unset($unguarded);
55
+
50
56
$file_phid = $file->getPHID();
51
57
}
52
58
···
55
61
'tooHuge' => $too_huge,
56
62
'filePHID' => $file_phid,
57
63
);
58
-
}
59
-
60
-
private function newFile(DiffusionRequest $drequest, $content) {
61
-
$path = $drequest->getPath();
62
-
$name = basename($path);
63
-
64
-
$repository = $drequest->getRepository();
65
-
$repository_phid = $repository->getPHID();
66
-
67
-
$file = PhabricatorFile::buildFromFileDataOrHash(
68
-
$content,
69
-
array(
70
-
'name' => $name,
71
-
'ttl' => time() + phutil_units('48 hours in seconds'),
72
-
'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
73
-
));
74
-
75
-
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
76
-
$file->attachToObject($repository_phid);
77
-
unset($unguarded);
78
-
79
-
return $file;
80
64
}
81
65
82
66
}
+28
-5
src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php
+28
-5
src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php
···
54
54
$future->setStdoutSizeLimit($byte_limit + 1);
55
55
}
56
56
57
+
$drequest = $this->getRequest();
58
+
59
+
$name = basename($drequest->getPath());
60
+
$ttl = PhabricatorTime::getNow() + phutil_units('48 hours in seconds');
61
+
57
62
try {
58
-
$file_content = $this->resolveFileContentFuture($future);
63
+
$threshold = PhabricatorFileStorageEngine::getChunkThreshold();
64
+
$future->setReadBufferSize($threshold);
65
+
66
+
$source = id(new PhabricatorExecFutureFileUploadSource())
67
+
->setName($name)
68
+
->setTTL($ttl)
69
+
->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
70
+
->setExecFuture($future);
71
+
72
+
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
73
+
$file = $source->uploadFile();
74
+
unset($unguarded);
75
+
59
76
} catch (CommandException $ex) {
60
77
if (!$future->getWasKilledByTimeout()) {
61
78
throw $ex;
62
79
}
63
80
64
81
$this->didHitTimeLimit = true;
65
-
$file_content = null;
82
+
$file = null;
66
83
}
67
84
68
-
if ($byte_limit && (strlen($file_content) > $byte_limit)) {
85
+
if ($byte_limit && ($file->getByteSize() > $byte_limit)) {
69
86
$this->didHitByteLimit = true;
70
-
$file_content = null;
87
+
88
+
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
89
+
id(new PhabricatorDestructionEngine())
90
+
->destroyObject($file);
91
+
unset($unguarded);
92
+
93
+
$file = null;
71
94
}
72
95
73
-
return $file_content;
96
+
return $file;
74
97
}
75
98
76
99
}
+6
-14
src/applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php
+6
-14
src/applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php
···
26
26
$rows = array();
27
27
$rowc = array();
28
28
foreach ($engines as $key => $engine) {
29
-
$limited = $no;
29
+
if ($engine->isTestEngine()) {
30
+
continue;
31
+
}
32
+
30
33
$limit = null;
31
34
if ($engine->hasFilesizeLimit()) {
32
-
$limited = $yes;
33
35
$limit = phutil_format_bytes($engine->getFilesizeLimit());
36
+
} else {
37
+
$limit = pht('Unlimited');
34
38
}
35
39
36
40
if ($engine->canWriteFiles()) {
···
39
43
$writable = $no;
40
44
}
41
45
42
-
if ($engine->isTestEngine()) {
43
-
$test = $yes;
44
-
} else {
45
-
$test = $no;
46
-
}
47
-
48
46
if (isset($writable_engines[$key]) || isset($chunk_engines[$key])) {
49
47
$rowc[] = 'highlighted';
50
48
} else {
···
54
52
$rows[] = array(
55
53
$key,
56
54
get_class($engine),
57
-
$test,
58
55
$writable,
59
-
$limited,
60
56
$limit,
61
57
);
62
58
}
···
67
63
array(
68
64
pht('Key'),
69
65
pht('Class'),
70
-
pht('Unit Test'),
71
66
pht('Writable'),
72
-
pht('Has Limit'),
73
67
pht('Limit'),
74
68
))
75
69
->setRowClasses($rowc)
···
77
71
array(
78
72
'',
79
73
'wide',
80
-
'',
81
-
'',
82
74
'',
83
75
'n',
84
76
));
+28
src/applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php
+28
src/applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php
···
1
+
<?php
2
+
3
+
final class PhabricatorExecFutureFileUploadSource
4
+
extends PhabricatorFileUploadSource {
5
+
6
+
private $future;
7
+
8
+
public function setExecFuture(ExecFuture $future) {
9
+
$this->future = $future;
10
+
return $this;
11
+
}
12
+
13
+
public function getExecFuture() {
14
+
return $this->future;
15
+
}
16
+
17
+
protected function newDataIterator() {
18
+
$future = $this->getExecFuture();
19
+
20
+
return id(new LinesOfALargeExecFuture($future))
21
+
->setDelimiter(null);
22
+
}
23
+
24
+
protected function getDataLength() {
25
+
return null;
26
+
}
27
+
28
+
}
+235
src/applications/files/uploadsource/PhabricatorFileUploadSource.php
+235
src/applications/files/uploadsource/PhabricatorFileUploadSource.php
···
1
+
<?php
2
+
3
+
abstract class PhabricatorFileUploadSource
4
+
extends Phobject {
5
+
6
+
private $name;
7
+
private $ttl;
8
+
private $viewPolicy;
9
+
10
+
private $rope;
11
+
private $data;
12
+
private $shouldChunk;
13
+
private $didRewind;
14
+
private $totalBytesWritten = 0;
15
+
16
+
public function setName($name) {
17
+
$this->name = $name;
18
+
return $this;
19
+
}
20
+
21
+
public function getName() {
22
+
return $this->name;
23
+
}
24
+
25
+
public function setTTL($ttl) {
26
+
$this->ttl = $ttl;
27
+
return $this;
28
+
}
29
+
30
+
public function getTTL() {
31
+
return $this->ttl;
32
+
}
33
+
34
+
public function setViewPolicy($view_policy) {
35
+
$this->viewPolicy = $view_policy;
36
+
return $this;
37
+
}
38
+
39
+
public function getViewPolicy() {
40
+
return $this->viewPolicy;
41
+
}
42
+
43
+
public function uploadFile() {
44
+
if (!$this->shouldChunkFile()) {
45
+
return $this->writeSingleFile();
46
+
} else {
47
+
return $this->writeChunkedFile();
48
+
}
49
+
}
50
+
51
+
private function getDataIterator() {
52
+
if (!$this->data) {
53
+
$this->data = $this->newDataIterator();
54
+
}
55
+
return $this->data;
56
+
}
57
+
58
+
private function getRope() {
59
+
if (!$this->rope) {
60
+
$this->rope = new PhutilRope();
61
+
}
62
+
return $this->rope;
63
+
}
64
+
65
+
abstract protected function newDataIterator();
66
+
abstract protected function getDataLength();
67
+
68
+
private function readFileData() {
69
+
$data = $this->getDataIterator();
70
+
71
+
if (!$this->didRewind) {
72
+
$data->rewind();
73
+
$this->didRewind = true;
74
+
} else {
75
+
$data->next();
76
+
}
77
+
78
+
if (!$data->valid()) {
79
+
return false;
80
+
}
81
+
82
+
$rope = $this->getRope();
83
+
$rope->append($data->current());
84
+
85
+
return true;
86
+
}
87
+
88
+
private function shouldChunkFile() {
89
+
if ($this->shouldChunk !== null) {
90
+
return $this->shouldChunk;
91
+
}
92
+
93
+
$threshold = PhabricatorFileStorageEngine::getChunkThreshold();
94
+
95
+
// If we don't know how large the file is, we're going to read some data
96
+
// from it until we know whether it's a small file or not. This will give
97
+
// us enough information to make a decision about chunking.
98
+
$length = $this->getDataLength();
99
+
if ($length === null) {
100
+
$rope = $this->getRope();
101
+
while ($this->readFileData()) {
102
+
$length = $rope->getByteLength();
103
+
if ($length > $threshold) {
104
+
break;
105
+
}
106
+
}
107
+
}
108
+
109
+
$this->shouldChunk = ($length > $threshold);
110
+
111
+
return $this->shouldChunk;
112
+
}
113
+
114
+
private function writeSingleFile() {
115
+
while ($this->readFileData()) {
116
+
// Read the entire file.
117
+
}
118
+
119
+
$rope = $this->getRope();
120
+
$data = $rope->getAsString();
121
+
122
+
$parameters = $this->getNewFileParameters();
123
+
124
+
return PhabricatorFile::newFromFileData($data, $parameters);
125
+
}
126
+
127
+
private function writeChunkedFile() {
128
+
$engine = $this->getChunkEngine();
129
+
130
+
$parameters = $this->getNewFileParameters();
131
+
132
+
$parameters = array(
133
+
'isPartial' => true,
134
+
) + $parameters;
135
+
136
+
$data_length = $this->getDataLength();
137
+
if ($data_length !== null) {
138
+
$length = $data_length;
139
+
} else {
140
+
$length = 0;
141
+
}
142
+
143
+
$file = PhabricatorFile::newChunkedFile($engine, $length, $parameters);
144
+
$file->save();
145
+
146
+
$rope = $this->getRope();
147
+
148
+
// Read the source, writing chunks as we get enough data.
149
+
while ($this->readFileData()) {
150
+
while (true) {
151
+
$rope_length = $rope->getByteLength();
152
+
if ($rope_length < $engine->getChunkSize()) {
153
+
break;
154
+
}
155
+
$this->writeChunk($file, $engine);
156
+
}
157
+
}
158
+
159
+
// If we have extra bytes at the end, write them.
160
+
if ($rope->getByteLength()) {
161
+
$this->writeChunk($file, $engine);
162
+
}
163
+
164
+
$file->setIsPartial(0);
165
+
if ($data_length === null) {
166
+
$file->setByteSize($this->getTotalBytesWritten());
167
+
}
168
+
$file->save();
169
+
170
+
return $file;
171
+
}
172
+
173
+
private function writeChunk(
174
+
PhabricatorFile $file,
175
+
PhabricatorFileStorageEngine $engine) {
176
+
177
+
$offset = $this->getTotalBytesWritten();
178
+
$max_length = $engine->getChunkSize();
179
+
$rope = $this->getRope();
180
+
181
+
$data = $rope->getPrefixBytes($max_length);
182
+
$actual_length = strlen($data);
183
+
$rope->removeBytesFromHead($actual_length);
184
+
185
+
$chunk_data = PhabricatorFile::newFromFileData(
186
+
$data,
187
+
array(
188
+
'name' => $file->getMonogram().'.chunk-'.$offset,
189
+
'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
190
+
));
191
+
192
+
$chunk = PhabricatorFileChunk::initializeNewChunk(
193
+
$file->getStorageHandle(),
194
+
$offset,
195
+
$offset + $actual_length);
196
+
197
+
$chunk
198
+
->setDataFilePHID($chunk_data->getPHID())
199
+
->save();
200
+
201
+
$this->setTotalBytesWritten($offset + $actual_length);
202
+
203
+
return $chunk;
204
+
}
205
+
206
+
private function getNewFileParameters() {
207
+
return array(
208
+
'name' => $this->getName(),
209
+
'ttl' => $this->getTTL(),
210
+
'viewPolicy' => $this->getViewPolicy(),
211
+
);
212
+
}
213
+
214
+
private function getChunkEngine() {
215
+
$chunk_engines = PhabricatorFileStorageEngine::loadWritableChunkEngines();
216
+
if (!$chunk_engines) {
217
+
throw new Exception(
218
+
pht(
219
+
'Unable to upload file: this server is not configured with any '.
220
+
'storage engine which can store large files.'));
221
+
}
222
+
223
+
return head($chunk_engines);
224
+
}
225
+
226
+
private function setTotalBytesWritten($total_bytes_written) {
227
+
$this->totalBytesWritten = $total_bytes_written;
228
+
return $this;
229
+
}
230
+
231
+
private function getTotalBytesWritten() {
232
+
return $this->totalBytesWritten;
233
+
}
234
+
235
+
}