@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add support to Files for file storage formats, to support encryption-at-rest
Summary:
Ref T11140. When reading and writing files, we optionally apply a "storage format" to them.
The default format is "raw", which means we just store the raw data.
This change modularizes formats and adds a "rot13" format, which proves formatting works and is testable. In the future, I'll add real encryption formats.
Test Plan:
- Added unit tests.
- Viewed files in web UI.
- Changed a file's format to rot13, saw the data get rotated on display.
- Set default format to rot13:
- Uploaded a small file, verified data was stored as rot13.
- Uploaded a large file, verified metadata was stored as "raw" (just a type, no actual data) and blob data was stored as rot13.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T11140
Differential Revision: https://secure.phabricator.com/D16122
+244
-31
+8
src/__phutil_library_map__.php
+8
src/__phutil_library_map__.php
···
2490
2490
'PhabricatorFileLinkView' => 'view/layout/PhabricatorFileLinkView.php',
2491
2491
'PhabricatorFileListController' => 'applications/files/controller/PhabricatorFileListController.php',
2492
2492
'PhabricatorFileQuery' => 'applications/files/query/PhabricatorFileQuery.php',
2493
+
'PhabricatorFileROT13StorageFormat' => 'applications/files/format/PhabricatorFileROT13StorageFormat.php',
2494
+
'PhabricatorFileRawStorageFormat' => 'applications/files/format/PhabricatorFileRawStorageFormat.php',
2493
2495
'PhabricatorFileSchemaSpec' => 'applications/files/storage/PhabricatorFileSchemaSpec.php',
2494
2496
'PhabricatorFileSearchEngine' => 'applications/files/query/PhabricatorFileSearchEngine.php',
2495
2497
'PhabricatorFileStorageBlob' => 'applications/files/storage/PhabricatorFileStorageBlob.php',
2496
2498
'PhabricatorFileStorageConfigurationException' => 'applications/files/exception/PhabricatorFileStorageConfigurationException.php',
2497
2499
'PhabricatorFileStorageEngine' => 'applications/files/engine/PhabricatorFileStorageEngine.php',
2498
2500
'PhabricatorFileStorageEngineTestCase' => 'applications/files/engine/__tests__/PhabricatorFileStorageEngineTestCase.php',
2501
+
'PhabricatorFileStorageFormat' => 'applications/files/format/PhabricatorFileStorageFormat.php',
2502
+
'PhabricatorFileStorageFormatTestCase' => 'applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php',
2499
2503
'PhabricatorFileTemporaryGarbageCollector' => 'applications/files/garbagecollector/PhabricatorFileTemporaryGarbageCollector.php',
2500
2504
'PhabricatorFileTestCase' => 'applications/files/storage/__tests__/PhabricatorFileTestCase.php',
2501
2505
'PhabricatorFileTestDataGenerator' => 'applications/files/lipsum/PhabricatorFileTestDataGenerator.php',
···
7134
7138
'PhabricatorFileLinkView' => 'AphrontView',
7135
7139
'PhabricatorFileListController' => 'PhabricatorFileController',
7136
7140
'PhabricatorFileQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
7141
+
'PhabricatorFileROT13StorageFormat' => 'PhabricatorFileStorageFormat',
7142
+
'PhabricatorFileRawStorageFormat' => 'PhabricatorFileStorageFormat',
7137
7143
'PhabricatorFileSchemaSpec' => 'PhabricatorConfigSchemaSpec',
7138
7144
'PhabricatorFileSearchEngine' => 'PhabricatorApplicationSearchEngine',
7139
7145
'PhabricatorFileStorageBlob' => 'PhabricatorFileDAO',
7140
7146
'PhabricatorFileStorageConfigurationException' => 'Exception',
7141
7147
'PhabricatorFileStorageEngine' => 'Phobject',
7142
7148
'PhabricatorFileStorageEngineTestCase' => 'PhabricatorTestCase',
7149
+
'PhabricatorFileStorageFormat' => 'Phobject',
7150
+
'PhabricatorFileStorageFormatTestCase' => 'PhabricatorTestCase',
7143
7151
'PhabricatorFileTemporaryGarbageCollector' => 'PhabricatorGarbageCollector',
7144
7152
'PhabricatorFileTestCase' => 'PhabricatorTestCase',
7145
7153
'PhabricatorFileTestDataGenerator' => 'PhabricatorTestDataGenerator',
+12
-5
src/applications/files/controller/PhabricatorFileInfoController.php
+12
-5
src/applications/files/controller/PhabricatorFileInfoController.php
···
256
256
$types[] = pht('Profile');
257
257
}
258
258
259
-
$types = implode(', ', $types);
260
-
$finfo->addProperty(pht('Attributes'), $types);
259
+
if ($types) {
260
+
$types = implode(', ', $types);
261
+
$finfo->addProperty(pht('Attributes'), $types);
262
+
}
261
263
262
264
$storage_properties = new PHUIPropertyListView();
263
265
$box->addPropertyList($storage_properties, pht('Storage'));
···
266
268
pht('Engine'),
267
269
$file->getStorageEngine());
268
270
269
-
$storage_properties->addProperty(
270
-
pht('Format'),
271
-
$file->getStorageFormat());
271
+
$format_key = $file->getStorageFormat();
272
+
$format = PhabricatorFileStorageFormat::getFormat($format_key);
273
+
if ($format) {
274
+
$format_name = $format->getStorageFormatName();
275
+
} else {
276
+
$format_name = pht('Unknown ("%s")', $format_key);
277
+
}
278
+
$storage_properties->addProperty(pht('Format'), $format_name);
272
279
273
280
$storage_properties->addProperty(
274
281
pht('Handle'),
+1
-1
src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php
+1
-1
src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php
···
174
174
return (4 * 1024 * 1024);
175
175
}
176
176
177
-
public function getFileDataIterator(PhabricatorFile $file, $begin, $end) {
177
+
public function getRawFileDataIterator(PhabricatorFile $file, $begin, $end) {
178
178
$chunks = id(new PhabricatorFileChunkQuery())
179
179
->setViewer(PhabricatorUser::getOmnipotentUser())
180
180
->withChunkHandles(array($file->getStorageHandle()))
+2
-2
src/applications/files/engine/PhabricatorFileStorageEngine.php
+2
-2
src/applications/files/engine/PhabricatorFileStorageEngine.php
···
325
325
return $engine->getChunkSize();
326
326
}
327
327
328
-
public function getFileDataIterator(PhabricatorFile $file, $begin, $end) {
328
+
public function getRawFileDataIterator(PhabricatorFile $file, $begin, $end) {
329
329
// The default implementation is trivial and just loads the entire file
330
330
// upfront.
331
-
$data = $file->loadFileData();
331
+
$data = $this->readFile($file->getStorageHandle());
332
332
333
333
if ($begin !== null && $end !== null) {
334
334
$data = substr($data, $begin, ($end - $begin));
+44
src/applications/files/format/PhabricatorFileROT13StorageFormat.php
+44
src/applications/files/format/PhabricatorFileROT13StorageFormat.php
···
1
+
<?php
2
+
3
+
/**
4
+
* Trivial example of a file storage format for at-rest encryption.
5
+
*
6
+
* This format applies ROT13 encoding to file data as it is stored and
7
+
* reverses it on the way out. This encoding is trivially reversible. This
8
+
* format is for testing, developing, and understanding encoding formats and
9
+
* is not intended for production use.
10
+
*/
11
+
final class PhabricatorFileROT13StorageFormat
12
+
extends PhabricatorFileStorageFormat {
13
+
14
+
const FORMATKEY = 'rot13';
15
+
16
+
public function getStorageFormatName() {
17
+
return pht('Encoded (ROT13)');
18
+
}
19
+
20
+
public function newReadIterator($raw_iterator) {
21
+
$file = $this->getFile();
22
+
$iterations = $file->getStorageProperty('iterations', 1);
23
+
24
+
$value = $file->loadDataFromIterator($raw_iterator);
25
+
for ($ii = 0; $ii < $iterations; $ii++) {
26
+
$value = str_rot13($value);
27
+
}
28
+
29
+
return array($value);
30
+
}
31
+
32
+
public function newWriteIterator($raw_iterator) {
33
+
return $this->newReadIterator($raw_iterator);
34
+
}
35
+
36
+
public function newStorageProperties() {
37
+
// For extreme security, repeatedly encode the data using a random (odd)
38
+
// number of iterations.
39
+
return array(
40
+
'iterations' => (mt_rand(1, 3) * 2) - 1,
41
+
);
42
+
}
43
+
44
+
}
+20
src/applications/files/format/PhabricatorFileRawStorageFormat.php
+20
src/applications/files/format/PhabricatorFileRawStorageFormat.php
···
1
+
<?php
2
+
3
+
final class PhabricatorFileRawStorageFormat
4
+
extends PhabricatorFileStorageFormat {
5
+
6
+
const FORMATKEY = 'raw';
7
+
8
+
public function getStorageFormatName() {
9
+
return pht('Raw Data');
10
+
}
11
+
12
+
public function newReadIterator($raw_iterator) {
13
+
return $raw_iterator;
14
+
}
15
+
16
+
public function newWriteIterator($raw_iterator) {
17
+
return $raw_iterator;
18
+
}
19
+
20
+
}
+58
src/applications/files/format/PhabricatorFileStorageFormat.php
+58
src/applications/files/format/PhabricatorFileStorageFormat.php
···
1
+
<?php
2
+
3
+
abstract class PhabricatorFileStorageFormat
4
+
extends Phobject {
5
+
6
+
private $file;
7
+
8
+
final public function setFile(PhabricatorFile $file) {
9
+
$this->file = $file;
10
+
return $this;
11
+
}
12
+
13
+
final public function getFile() {
14
+
if (!$this->file) {
15
+
throw new PhutilInvalidStateException('setFile');
16
+
}
17
+
return $this->file;
18
+
}
19
+
20
+
abstract public function getStorageFormatName();
21
+
22
+
abstract public function newReadIterator($raw_iterator);
23
+
abstract public function newWriteIterator($raw_iterator);
24
+
25
+
public function newStorageProperties() {
26
+
return array();
27
+
}
28
+
29
+
final public function getStorageFormatKey() {
30
+
return $this->getPhobjectClassConstant('FORMATKEY');
31
+
}
32
+
33
+
final public static function getAllFormats() {
34
+
return id(new PhutilClassMapQuery())
35
+
->setAncestorClass(__CLASS__)
36
+
->setUniqueMethod('getStorageFormatKey')
37
+
->execute();
38
+
}
39
+
40
+
final public static function getFormat($key) {
41
+
$formats = self::getAllFormats();
42
+
return idx($formats, $key);
43
+
}
44
+
45
+
final public static function requireFormat($key) {
46
+
$format = self::getFormat($key);
47
+
48
+
if (!$format) {
49
+
throw new Exception(
50
+
pht(
51
+
'No file storage format with key "%s" exists.',
52
+
$key));
53
+
}
54
+
55
+
return $format;
56
+
}
57
+
58
+
}
+38
src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php
+38
src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php
···
1
+
<?php
2
+
3
+
final class PhabricatorFileStorageFormatTestCase extends PhabricatorTestCase {
4
+
5
+
protected function getPhabricatorTestCaseConfiguration() {
6
+
return array(
7
+
self::PHABRICATOR_TESTCONFIG_BUILD_STORAGE_FIXTURES => true,
8
+
);
9
+
}
10
+
11
+
public function testRot13Storage() {
12
+
$engine = new PhabricatorTestStorageEngine();
13
+
$rot13_format = PhabricatorFileROT13StorageFormat::FORMATKEY;
14
+
15
+
$data = 'The cow jumped over the full moon.';
16
+
$expect = 'Gur pbj whzcrq bire gur shyy zbba.';
17
+
18
+
$params = array(
19
+
'name' => 'test.dat',
20
+
'storageEngines' => array(
21
+
$engine,
22
+
),
23
+
'format' => $rot13_format,
24
+
);
25
+
26
+
$file = PhabricatorFile::newFromFileData($data, $params);
27
+
28
+
// We should have a file stored as rot13, which reads back the input
29
+
// data correctly.
30
+
$this->assertEqual($rot13_format, $file->getStorageFormat());
31
+
$this->assertEqual($data, $file->loadFileData());
32
+
33
+
// The actual raw data in the storage engine should be encoded.
34
+
$raw_data = $engine->readFile($file->getStorageHandle());
35
+
$this->assertEqual($expect, $raw_data);
36
+
}
37
+
38
+
}
+61
-23
src/applications/files/storage/PhabricatorFile.php
+61
-23
src/applications/files/storage/PhabricatorFile.php
···
26
26
PhabricatorPolicyInterface,
27
27
PhabricatorDestructibleInterface {
28
28
29
-
const STORAGE_FORMAT_RAW = 'raw';
30
-
31
29
const METADATA_IMAGE_WIDTH = 'width';
32
30
const METADATA_IMAGE_HEIGHT = 'height';
33
31
const METADATA_CAN_CDN = 'canCDN';
34
32
const METADATA_BUILTIN = 'builtin';
35
33
const METADATA_PARTIAL = 'partial';
36
34
const METADATA_PROFILE = 'profile';
35
+
const METADATA_STORAGE = 'storage';
37
36
38
37
protected $name;
39
38
protected $mimeType;
···
233
232
$hash);
234
233
235
234
if ($file) {
236
-
// copy storageEngine, storageHandle, storageFormat
237
235
$copy_of_storage_engine = $file->getStorageEngine();
238
236
$copy_of_storage_handle = $file->getStorageHandle();
239
237
$copy_of_storage_format = $file->getStorageFormat();
238
+
$copy_of_storage_properties = $file->getStorageProperties();
240
239
$copy_of_byte_size = $file->getByteSize();
241
240
$copy_of_mime_type = $file->getMimeType();
242
241
···
248
247
$new_file->setStorageEngine($copy_of_storage_engine);
249
248
$new_file->setStorageHandle($copy_of_storage_handle);
250
249
$new_file->setStorageFormat($copy_of_storage_format);
250
+
$new_file->setStorageProperties($copy_of_storage_properties);
251
251
$new_file->setMimeType($copy_of_mime_type);
252
252
$new_file->copyDimensions($file);
253
253
···
290
290
291
291
$file->setStorageEngine($engine->getEngineIdentifier());
292
292
$file->setStorageHandle(PhabricatorFileChunk::newChunkHandle());
293
-
$file->setStorageFormat(self::STORAGE_FORMAT_RAW);
293
+
294
+
// Chunked files are always stored raw because they do not actually store
295
+
// data. The chunks do, and can be individually formatted.
296
+
$file->setStorageFormat(PhabricatorFileRawStorageFormat::FORMATKEY);
297
+
294
298
$file->setIsPartial(1);
295
299
296
300
$file->readPropertiesFromParameters($params);
···
322
326
323
327
$file = self::initializeNewFile();
324
328
329
+
$default_key = PhabricatorFileRawStorageFormat::FORMATKEY;
330
+
$format_key = idx($params, 'format', $default_key);
331
+
332
+
$format = id(clone PhabricatorFileStorageFormat::requireFormat($format_key))
333
+
->setFile($file);
334
+
335
+
$properties = $format->newStorageProperties();
336
+
$file->setStorageFormat($format->getStorageFormatKey());
337
+
$file->setStorageProperties($properties);
338
+
325
339
$data_handle = null;
326
340
$engine_identifier = null;
327
341
$exceptions = array();
···
360
374
361
375
$file->setStorageEngine($engine_identifier);
362
376
$file->setStorageHandle($data_handle);
363
-
364
-
// TODO: This is probably YAGNI, but allows for us to do encryption or
365
-
// compression later if we want.
366
-
$file->setStorageFormat(self::STORAGE_FORMAT_RAW);
367
377
368
378
$file->readPropertiesFromParameters($params);
369
379
···
434
444
435
445
$engine_class = get_class($engine);
436
446
437
-
$data_handle = $engine->writeFile($data, $params);
447
+
$key = $this->getStorageFormat();
448
+
$format = id(clone PhabricatorFileStorageFormat::requireFormat($key))
449
+
->setFile($this);
450
+
451
+
$data_iterator = array($data);
452
+
$formatted_iterator = $format->newWriteIterator($data_iterator);
453
+
$formatted_data = $this->loadDataFromIterator($formatted_iterator);
454
+
455
+
$data_handle = $engine->writeFile($formatted_data, $params);
438
456
439
457
if (!$data_handle || strlen($data_handle) > 255) {
440
458
// This indicates an improperly implemented storage engine.
···
663
681
}
664
682
665
683
public function loadFileData() {
666
-
667
-
$engine = $this->instantiateStorageEngine();
668
-
$data = $engine->readFile($this->getStorageHandle());
669
-
670
-
switch ($this->getStorageFormat()) {
671
-
case self::STORAGE_FORMAT_RAW:
672
-
$data = $data;
673
-
break;
674
-
default:
675
-
throw new Exception(pht('Unknown storage format.'));
676
-
}
677
-
678
-
return $data;
684
+
$iterator = $this->getFileDataIterator();
685
+
return $this->loadDataFromIterator($iterator);
679
686
}
680
687
681
688
···
688
695
*/
689
696
public function getFileDataIterator($begin = null, $end = null) {
690
697
$engine = $this->instantiateStorageEngine();
691
-
return $engine->getFileDataIterator($this, $begin, $end);
698
+
$raw_iterator = $engine->getRawFileDataIterator($this, $begin, $end);
699
+
700
+
$key = $this->getStorageFormat();
701
+
702
+
$format = id(clone PhabricatorFileStorageFormat::requireFormat($key))
703
+
->setFile($this);
704
+
705
+
return $format->newReadIterator($raw_iterator);
692
706
}
693
707
694
708
···
915
929
916
930
public function generateSecretKey() {
917
931
return Filesystem::readRandomCharacters(20);
932
+
}
933
+
934
+
public function setStorageProperties(array $properties) {
935
+
$this->metadata[self::METADATA_STORAGE] = $properties;
936
+
return $this;
937
+
}
938
+
939
+
public function getStorageProperties() {
940
+
return idx($this->metadata, self::METADATA_STORAGE, array());
941
+
}
942
+
943
+
public function getStorageProperty($key, $default = null) {
944
+
$properties = $this->getStorageProperties();
945
+
return idx($properties, $key, $default);
946
+
}
947
+
948
+
public function loadDataFromIterator($iterator) {
949
+
$result = '';
950
+
951
+
foreach ($iterator as $chunk) {
952
+
$result .= $chunk;
953
+
}
954
+
955
+
return $result;
918
956
}
919
957
920
958
public function updateDimensions($save = true) {