@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Remove SHA1 file content hashing and make Files work without any hashing
Summary:
Ref T12464. We currently use SHA1 to detect when two files have the same content so we don't have to store two copies of the data.
Now that a SHA1 collision is known, this is theoretically dangerous. T12464 describes the shape of a possible attack.
Before replacing this with something more robust, shore things up so things work correctly if we don't hash at all. This mechanism is entirely optional; it only helps us store less data if some files are duplicates.
(This mechanism is also less important now than it once was, before we added temporary files.)
Test Plan: Uploaded multiple identical files, saw the uploads work and the files store separate copies of the same data.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T12464
Differential Revision: https://secure.phabricator.com/D17619
+63
-43
+7
-4
src/applications/files/conduit/FileAllocateConduitAPIMethod.php
+7
-4
src/applications/files/conduit/FileAllocateConduitAPIMethod.php
···
36
36
$properties = array(
37
37
'name' => $name,
38
38
'authorPHID' => $viewer->getPHID(),
39
-
'viewPolicy' => $view_policy,
40
39
'isExplicitUpload' => true,
41
40
);
41
+
42
+
if ($view_policy !== null) {
43
+
$properties['viewPolicy'] = $view_policy;
44
+
}
42
45
43
46
$ttl = $request->getValue('deleteAfterEpoch');
44
47
if ($ttl) {
···
46
49
}
47
50
48
51
$file = null;
49
-
if ($hash) {
52
+
if ($hash !== null) {
50
53
$file = PhabricatorFile::newFileFromContentHash(
51
54
$hash,
52
55
$properties);
53
56
}
54
57
55
-
if ($hash && !$file) {
58
+
if ($hash !== null && !$file) {
56
59
$chunked_hash = PhabricatorChunkedFileStorageEngine::getChunkedHash(
57
60
$viewer,
58
61
$hash);
···
103
106
if ($chunk_engines) {
104
107
$chunk_properties = $properties;
105
108
106
-
if ($hash) {
109
+
if ($hash !== null) {
107
110
$chunk_properties += array(
108
111
'chunkedHash' => $chunked_hash,
109
112
);
+16
-10
src/applications/files/conduit/FileUploadConduitAPIMethod.php
+16
-10
src/applications/files/conduit/FileUploadConduitAPIMethod.php
···
27
27
$viewer = $request->getUser();
28
28
29
29
$name = $request->getValue('name');
30
-
$can_cdn = $request->getValue('canCDN');
30
+
$can_cdn = (bool)$request->getValue('canCDN');
31
31
$view_policy = $request->getValue('viewPolicy');
32
32
33
33
$data = $request->getValue('data_base64');
34
34
$data = $this->decodeBase64($data);
35
35
36
-
$file = PhabricatorFile::newFromFileData(
37
-
$data,
38
-
array(
39
-
'name' => $name,
40
-
'authorPHID' => $viewer->getPHID(),
41
-
'viewPolicy' => $view_policy,
42
-
'canCDN' => $can_cdn,
43
-
'isExplicitUpload' => true,
44
-
));
36
+
$params = array(
37
+
'authorPHID' => $viewer->getPHID(),
38
+
'canCDN' => $can_cdn,
39
+
'isExplicitUpload' => true,
40
+
);
41
+
42
+
if ($name !== null) {
43
+
$params['name'] = $name;
44
+
}
45
+
46
+
if ($view_policy !== null) {
47
+
$params['viewPolicy'] = $view_policy;
48
+
}
49
+
50
+
$file = PhabricatorFile::newFromFileData($data, $params);
45
51
46
52
return $file->getPHID();
47
53
}
+35
-29
src/applications/files/storage/PhabricatorFile.php
+35
-29
src/applications/files/storage/PhabricatorFile.php
···
198
198
199
199
200
200
public static function newFileFromContentHash($hash, array $params) {
201
-
// Check to see if a file with same contentHash exist
201
+
if ($hash === null) {
202
+
return null;
203
+
}
204
+
205
+
// Check to see if a file with same hash already exists.
202
206
$file = id(new PhabricatorFile())->loadOneWhere(
203
207
'contentHash = %s LIMIT 1',
204
208
$hash);
209
+
if (!$file) {
210
+
return null;
211
+
}
205
212
206
-
if ($file) {
207
-
$copy_of_storage_engine = $file->getStorageEngine();
208
-
$copy_of_storage_handle = $file->getStorageHandle();
209
-
$copy_of_storage_format = $file->getStorageFormat();
210
-
$copy_of_storage_properties = $file->getStorageProperties();
211
-
$copy_of_byte_size = $file->getByteSize();
212
-
$copy_of_mime_type = $file->getMimeType();
213
+
$copy_of_storage_engine = $file->getStorageEngine();
214
+
$copy_of_storage_handle = $file->getStorageHandle();
215
+
$copy_of_storage_format = $file->getStorageFormat();
216
+
$copy_of_storage_properties = $file->getStorageProperties();
217
+
$copy_of_byte_size = $file->getByteSize();
218
+
$copy_of_mime_type = $file->getMimeType();
213
219
214
-
$new_file = self::initializeNewFile();
220
+
$new_file = self::initializeNewFile();
215
221
216
-
$new_file->setByteSize($copy_of_byte_size);
222
+
$new_file->setByteSize($copy_of_byte_size);
217
223
218
-
$new_file->setContentHash($hash);
219
-
$new_file->setStorageEngine($copy_of_storage_engine);
220
-
$new_file->setStorageHandle($copy_of_storage_handle);
221
-
$new_file->setStorageFormat($copy_of_storage_format);
222
-
$new_file->setStorageProperties($copy_of_storage_properties);
223
-
$new_file->setMimeType($copy_of_mime_type);
224
-
$new_file->copyDimensions($file);
225
-
226
-
$new_file->readPropertiesFromParameters($params);
224
+
$new_file->setContentHash($hash);
225
+
$new_file->setStorageEngine($copy_of_storage_engine);
226
+
$new_file->setStorageHandle($copy_of_storage_handle);
227
+
$new_file->setStorageFormat($copy_of_storage_format);
228
+
$new_file->setStorageProperties($copy_of_storage_properties);
229
+
$new_file->setMimeType($copy_of_mime_type);
230
+
$new_file->copyDimensions($file);
227
231
228
-
$new_file->save();
232
+
$new_file->readPropertiesFromParameters($params);
229
233
230
-
return $new_file;
231
-
}
234
+
$new_file->save();
232
235
233
-
return $file;
236
+
return $new_file;
234
237
}
235
238
236
239
public static function newChunkedFile(
···
353
356
}
354
357
355
358
$file->setByteSize(strlen($data));
356
-
$file->setContentHash(self::hashFileContent($data));
359
+
360
+
$hash = self::hashFileContent($data);
361
+
$file->setContentHash($hash);
357
362
358
363
$file->setStorageEngine($engine_identifier);
359
364
$file->setStorageHandle($data_handle);
···
379
384
380
385
public static function newFromFileData($data, array $params = array()) {
381
386
$hash = self::hashFileContent($data);
382
-
$file = self::newFileFromContentHash($hash, $params);
383
387
384
-
if ($file) {
385
-
return $file;
388
+
if ($hash !== null) {
389
+
$file = self::newFileFromContentHash($hash, $params);
390
+
if ($file) {
391
+
return $file;
392
+
}
386
393
}
387
394
388
395
return self::buildFromFileData($data, $params);
···
710
717
}
711
718
}
712
719
713
-
714
720
public static function hashFileContent($data) {
715
-
return sha1($data);
721
+
return null;
716
722
}
717
723
718
724
public function loadFileData() {
+5
src/applications/files/storage/__tests__/PhabricatorFileTestCase.php
+5
src/applications/files/storage/__tests__/PhabricatorFileTestCase.php
···
301
301
302
302
$data = Filesystem::readRandomCharacters(64);
303
303
304
+
$hash = PhabricatorFile::hashFileContent($data);
305
+
if ($hash === null) {
306
+
$this->assertSkipped(pht('File content hashing is not available.'));
307
+
}
308
+
304
309
$params = array(
305
310
'name' => 'test.dat',
306
311
'storageEngines' => array(