@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Support resuming JS uploads of chunked files
Summary: Ref T7149. We can't compute hashes of large files efficiently, but we can resume uploads by the same author, with the same name and file size, which are only partially completed. This seems like a reasonable heuristic that is unlikely to ever misfire, even if it's a little magical.
Test Plan:
- Forced chunking on.
- Started uploading a chunked file.
- Closed the browser window.
- Dropped it into a new window.
- Upload resumed //(!!!)//
- Did this again.
- Downloaded the final file, which successfully reconstructed the original file.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: joshuaspence, chad, epriestley
Maniphest Tasks: T7149
Differential Revision: https://secure.phabricator.com/D12070
+101
-30
+2
-2
resources/celerity/map.php
+2
-2
resources/celerity/map.php
···
44
44
'rsrc/css/application/config/config-welcome.css' => '6abd79be',
45
45
'rsrc/css/application/config/setup-issue.css' => '22270af2',
46
46
'rsrc/css/application/config/unhandled-exception.css' => '37d4f9a2',
47
-
'rsrc/css/application/conpherence/durable-column.css' => '1f5c64e8',
47
+
'rsrc/css/application/conpherence/durable-column.css' => '8c951609',
48
48
'rsrc/css/application/conpherence/menu.css' => 'c6ac5299',
49
49
'rsrc/css/application/conpherence/message-pane.css' => '5930260a',
50
50
'rsrc/css/application/conpherence/notification.css' => '04a6e10a',
···
513
513
'changeset-view-manager' => '88be0133',
514
514
'config-options-css' => '7fedf08b',
515
515
'config-welcome-css' => '6abd79be',
516
-
'conpherence-durable-column-view' => '1f5c64e8',
516
+
'conpherence-durable-column-view' => '8c951609',
517
517
'conpherence-menu-css' => 'c6ac5299',
518
518
'conpherence-message-pane-css' => '5930260a',
519
519
'conpherence-notification-css' => '04a6e10a',
+29
-16
src/applications/files/conduit/FileAllocateConduitAPIMethod.php
+29
-16
src/applications/files/conduit/FileAllocateConduitAPIMethod.php
···
37
37
$hash = $request->getValue('contentHash');
38
38
$name = $request->getValue('name');
39
39
$view_policy = $request->getValue('viewPolicy');
40
-
$content_length = $request->getValue('contentLength');
40
+
$length = $request->getValue('contentLength');
41
41
42
42
$force_chunking = $request->getValue('forceChunking');
43
43
···
48
48
'isExplicitUpload' => true,
49
49
);
50
50
51
+
$file = null;
51
52
if ($hash) {
52
53
$file = PhabricatorFile::newFileFromContentHash(
53
54
$hash,
54
55
$properties);
55
-
56
-
if ($file) {
57
-
return array(
58
-
'upload' => false,
59
-
'filePHID' => $file->getPHID(),
60
-
);
61
-
}
56
+
}
62
57
58
+
if ($hash && !$file) {
63
59
$chunked_hash = PhabricatorChunkedFileStorageEngine::getChunkedHash(
64
60
$viewer,
65
61
$hash);
···
67
63
->setViewer($viewer)
68
64
->withContentHashes(array($chunked_hash))
69
65
->executeOne();
66
+
}
70
67
71
-
if ($file) {
72
-
return array(
73
-
'upload' => (bool)$file->getIsPartial(),
74
-
'filePHID' => $file->getPHID(),
75
-
);
68
+
if (strlen($name) && !$hash && !$file) {
69
+
if ($length > PhabricatorFileStorageEngine::getChunkThreshold()) {
70
+
// If we don't have a hash, but this file is large enough to store in
71
+
// chunks and thus may be resumable, try to find a partially uploaded
72
+
// file by the same author with the same name and same length. This
73
+
// allows us to resume uploads in Javascript where we can't efficiently
74
+
// compute file hashes.
75
+
$file = id(new PhabricatorFileQuery())
76
+
->setViewer($viewer)
77
+
->withAuthorPHIDs(array($viewer->getPHID()))
78
+
->withNames(array($name))
79
+
->withLengthBetween($length, $length)
80
+
->withIsPartial(true)
81
+
->setLimit(1)
82
+
->executeOne();
76
83
}
77
84
}
78
85
79
-
$engines = PhabricatorFileStorageEngine::loadStorageEngines(
80
-
$content_length);
86
+
if ($file) {
87
+
return array(
88
+
'upload' => (bool)$file->getIsPartial(),
89
+
'filePHID' => $file->getPHID(),
90
+
);
91
+
}
92
+
93
+
$engines = PhabricatorFileStorageEngine::loadStorageEngines($length);
81
94
if ($engines) {
82
95
83
96
if ($force_chunking) {
···
111
124
);
112
125
}
113
126
114
-
$file = $engine->allocateChunks($content_length, $chunk_properties);
127
+
$file = $engine->allocateChunks($length, $chunk_properties);
115
128
116
129
return array(
117
130
'upload' => true,
+10
-1
src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php
+10
-1
src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php
···
105
105
}
106
106
107
107
$input = $viewer->getAccountSecret().':'.$hash.':'.$viewer->getPHID();
108
-
return PhabricatorHash::digest($input);
108
+
return self::getChunkedHashForInput($input);
109
+
}
110
+
111
+
public static function getChunkedHashForInput($input) {
112
+
$rehash = PhabricatorHash::digest($input);
113
+
114
+
// Add a suffix to identify this as a chunk hash.
115
+
$rehash = substr($rehash, 0, -2).'-C';
116
+
117
+
return $rehash;
109
118
}
110
119
111
120
public function allocateChunks($length, array $properties) {
+56
-8
src/applications/files/query/PhabricatorFileQuery.php
+56
-8
src/applications/files/query/PhabricatorFileQuery.php
···
11
11
private $dateCreatedAfter;
12
12
private $dateCreatedBefore;
13
13
private $contentHashes;
14
+
private $minLength;
15
+
private $maxLength;
16
+
private $names;
17
+
private $isPartial;
14
18
15
19
public function withIDs(array $ids) {
16
20
$this->ids = $ids;
···
89
93
return $this;
90
94
}
91
95
96
+
public function withLengthBetween($min, $max) {
97
+
$this->minLength = $min;
98
+
$this->maxLength = $max;
99
+
return $this;
100
+
}
101
+
102
+
public function withNames(array $names) {
103
+
$this->names = $names;
104
+
return $this;
105
+
}
106
+
107
+
public function withIsPartial($partial) {
108
+
$this->isPartial = $partial;
109
+
return $this;
110
+
}
111
+
92
112
public function showOnlyExplicitUploads($explicit_uploads) {
93
113
$this->explicitUploads = $explicit_uploads;
94
114
return $this;
···
213
233
214
234
$where[] = $this->buildPagingClause($conn_r);
215
235
216
-
if ($this->ids) {
236
+
if ($this->ids !== null) {
217
237
$where[] = qsprintf(
218
238
$conn_r,
219
239
'f.id IN (%Ld)',
220
240
$this->ids);
221
241
}
222
242
223
-
if ($this->phids) {
243
+
if ($this->phids !== null) {
224
244
$where[] = qsprintf(
225
245
$conn_r,
226
246
'f.phid IN (%Ls)',
227
247
$this->phids);
228
248
}
229
249
230
-
if ($this->authorPHIDs) {
250
+
if ($this->authorPHIDs !== null) {
231
251
$where[] = qsprintf(
232
252
$conn_r,
233
253
'f.authorPHID IN (%Ls)',
234
254
$this->authorPHIDs);
235
255
}
236
256
237
-
if ($this->explicitUploads) {
257
+
if ($this->explicitUploads !== null) {
238
258
$where[] = qsprintf(
239
259
$conn_r,
240
260
'f.isExplicitUpload = true');
241
261
}
242
262
243
-
if ($this->transforms) {
263
+
if ($this->transforms !== null) {
244
264
$clauses = array();
245
265
foreach ($this->transforms as $transform) {
246
266
if ($transform['transform'] === true) {
···
259
279
$where[] = qsprintf($conn_r, '(%Q)', implode(') OR (', $clauses));
260
280
}
261
281
262
-
if ($this->dateCreatedAfter) {
282
+
if ($this->dateCreatedAfter !== null) {
263
283
$where[] = qsprintf(
264
284
$conn_r,
265
285
'f.dateCreated >= %d',
266
286
$this->dateCreatedAfter);
267
287
}
268
288
269
-
if ($this->dateCreatedBefore) {
289
+
if ($this->dateCreatedBefore !== null) {
270
290
$where[] = qsprintf(
271
291
$conn_r,
272
292
'f.dateCreated <= %d',
273
293
$this->dateCreatedBefore);
274
294
}
275
295
276
-
if ($this->contentHashes) {
296
+
if ($this->contentHashes !== null) {
277
297
$where[] = qsprintf(
278
298
$conn_r,
279
299
'f.contentHash IN (%Ls)',
280
300
$this->contentHashes);
301
+
}
302
+
303
+
if ($this->minLength !== null) {
304
+
$where[] = qsprintf(
305
+
$conn_r,
306
+
'byteSize >= %d',
307
+
$this->minLength);
308
+
}
309
+
310
+
if ($this->maxLength !== null) {
311
+
$where[] = qsprintf(
312
+
$conn_r,
313
+
'byteSize <= %d',
314
+
$this->maxLength);
315
+
}
316
+
317
+
if ($this->names !== null) {
318
+
$where[] = qsprintf(
319
+
$conn_r,
320
+
'name in (%Ls)',
321
+
$this->names);
322
+
}
323
+
324
+
if ($this->isPartial !== null) {
325
+
$where[] = qsprintf(
326
+
$conn_r,
327
+
'isPartial = %d',
328
+
(int)$this->isPartial);
281
329
}
282
330
283
331
return $this->formatWhereClause($where);
+4
-3
src/applications/files/storage/PhabricatorFile.php
+4
-3
src/applications/files/storage/PhabricatorFile.php
···
292
292
} else {
293
293
// See PhabricatorChunkedFileStorageEngine::getChunkedHash() for some
294
294
// discussion of this.
295
-
$file->setContentHash(
296
-
PhabricatorHash::digest(
297
-
Filesystem::readRandomBytes(64)));
295
+
$seed = Filesystem::readRandomBytes(64);
296
+
$hash = PhabricatorChunkedFileStorageEngine::getChunkedHashForInput(
297
+
$seed);
298
+
$file->setContentHash($hash);
298
299
}
299
300
300
301
$file->setStorageEngine($engine->getEngineIdentifier());