@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add `bin/files compact` for sharing file data storage
Summary:
Fixes T5912. When we write files, we attempt to share storage if two files have the same content.
In some cases, we may not share storage. Examples include:
- Files migrated with `bin/files migrate` (it's simpler not to try to dedupe them).
- Old files, from before storage was sharable (the mechanism did not exist).
- Files broken by the bug fixed in T5912.
Add a script to compact files by pointing files with the same content hash at the same file contnet.
In the particular case of files broken by the bug in T5912, we know the hash of the file's content and will only point them at a file that we can load the data for, so this fixes them.
Compaction is not hugely useful in general, but this script isn't too complex and the ability to fix damage from the bug in T5912 is desirable. We could remove this capability eventually.
Test Plan:
- Ran `files compact --all --dry-run` and sanity checked a bunch of the duplicates for actually being duplicates.
- Migrated individual files with `files compact Fnnn --trace` and verified the storage compacted and all files survived the process.
- Verified unused storage was correctly destroyed after removing the last reference to it.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: epriestley
Maniphest Tasks: T5912
Differential Revision: https://secure.phabricator.com/D10327
+137
-2
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
1578
1578
'PhabricatorFileUploadException' => 'applications/files/exception/PhabricatorFileUploadException.php',
1579
1579
'PhabricatorFilesApplication' => 'applications/files/application/PhabricatorFilesApplication.php',
1580
1580
'PhabricatorFilesConfigOptions' => 'applications/files/config/PhabricatorFilesConfigOptions.php',
1581
+
'PhabricatorFilesManagementCompactWorkflow' => 'applications/files/management/PhabricatorFilesManagementCompactWorkflow.php',
1581
1582
'PhabricatorFilesManagementEnginesWorkflow' => 'applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php',
1582
1583
'PhabricatorFilesManagementMigrateWorkflow' => 'applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php',
1583
1584
'PhabricatorFilesManagementPurgeWorkflow' => 'applications/files/management/PhabricatorFilesManagementPurgeWorkflow.php',
···
4430
4431
'PhabricatorFileUploadException' => 'Exception',
4431
4432
'PhabricatorFilesApplication' => 'PhabricatorApplication',
4432
4433
'PhabricatorFilesConfigOptions' => 'PhabricatorApplicationConfigOptions',
4434
+
'PhabricatorFilesManagementCompactWorkflow' => 'PhabricatorFilesManagementWorkflow',
4433
4435
'PhabricatorFilesManagementEnginesWorkflow' => 'PhabricatorFilesManagementWorkflow',
4434
4436
'PhabricatorFilesManagementMigrateWorkflow' => 'PhabricatorFilesManagementWorkflow',
4435
4437
'PhabricatorFilesManagementPurgeWorkflow' => 'PhabricatorFilesManagementWorkflow',
+133
src/applications/files/management/PhabricatorFilesManagementCompactWorkflow.php
+133
src/applications/files/management/PhabricatorFilesManagementCompactWorkflow.php
···
1
+
<?php
2
+
3
+
final class PhabricatorFilesManagementCompactWorkflow
4
+
extends PhabricatorFilesManagementWorkflow {
5
+
6
+
public function didConstruct() {
7
+
$this
8
+
->setName('compact')
9
+
->setSynopsis(
10
+
pht(
11
+
'Merge identical files to share the same storage. In some cases, '.
12
+
'this can repair files with missing data.'))
13
+
->setArguments(
14
+
array(
15
+
array(
16
+
'name' => 'dry-run',
17
+
'help' => pht('Show what would be compacted.'),
18
+
),
19
+
array(
20
+
'name' => 'all',
21
+
'help' => pht('Compact all files.'),
22
+
),
23
+
array(
24
+
'name' => 'names',
25
+
'wildcard' => true,
26
+
),
27
+
));
28
+
}
29
+
30
+
public function execute(PhutilArgumentParser $args) {
31
+
$console = PhutilConsole::getConsole();
32
+
33
+
$iterator = $this->buildIterator($args);
34
+
if (!$iterator) {
35
+
throw new PhutilArgumentUsageException(
36
+
pht(
37
+
'Either specify a list of files to compact, or use `--all` '.
38
+
'to compact all files.'));
39
+
}
40
+
41
+
$is_dry_run = $args->getArg('dry-run');
42
+
43
+
foreach ($iterator as $file) {
44
+
$monogram = $file->getMonogram();
45
+
46
+
$hash = $file->getContentHash();
47
+
if (!$hash) {
48
+
$console->writeOut(
49
+
"%s\n",
50
+
pht('%s: No content hash.', $monogram));
51
+
continue;
52
+
}
53
+
54
+
// Find other files with the same content hash. We're going to point
55
+
// them at the data for this file.
56
+
$similar_files = id(new PhabricatorFile())->loadAllWhere(
57
+
'contentHash = %s AND id != %d AND
58
+
(storageEngine != %s OR storageHandle != %s)',
59
+
$hash,
60
+
$file->getID(),
61
+
$file->getStorageEngine(),
62
+
$file->getStorageHandle());
63
+
if (!$similar_files) {
64
+
$console->writeOut(
65
+
"%s\n",
66
+
pht('%s: No other files with the same content hash.', $monogram));
67
+
continue;
68
+
}
69
+
70
+
// Only compact files into this one if we can load the data. This
71
+
// prevents us from breaking working files if we're missing some data.
72
+
try {
73
+
$data = $file->loadFileData();
74
+
} catch (Exception $ex) {
75
+
$data = null;
76
+
}
77
+
78
+
if ($data === null) {
79
+
$console->writeOut(
80
+
"%s\n",
81
+
pht(
82
+
'%s: Unable to load file data; declining to compact.',
83
+
$monogram));
84
+
continue;
85
+
}
86
+
87
+
foreach ($similar_files as $similar_file) {
88
+
if ($is_dry_run) {
89
+
$console->writeOut(
90
+
"%s\n",
91
+
pht(
92
+
'%s: Would compact storage with %s.',
93
+
$monogram,
94
+
$similar_file->getMonogram()));
95
+
continue;
96
+
}
97
+
98
+
$console->writeOut(
99
+
"%s\n",
100
+
pht(
101
+
'%s: Compacting storage with %s.',
102
+
$monogram,
103
+
$similar_file->getMonogram()));
104
+
105
+
$old_instance = null;
106
+
try {
107
+
$old_instance = $similar_file->instantiateStorageEngine();
108
+
$old_engine = $similar_file->getStorageEngine();
109
+
$old_handle = $similar_file->getStorageHandle();
110
+
} catch (Exception $ex) {
111
+
// If the old stuff is busted, we just won't try to delete the
112
+
// old data.
113
+
phlog($ex);
114
+
}
115
+
116
+
$similar_file
117
+
->setStorageEngine($file->getStorageEngine())
118
+
->setStorageHandle($file->getStorageHandle())
119
+
->save();
120
+
121
+
if ($old_instance) {
122
+
$similar_file->deleteFileDataIfUnused(
123
+
$old_instance,
124
+
$old_engine,
125
+
$old_handle);
126
+
}
127
+
}
128
+
}
129
+
130
+
return 0;
131
+
}
132
+
133
+
}
+2
-2
src/applications/files/storage/PhabricatorFile.php
+2
-2
src/applications/files/storage/PhabricatorFile.php
···
455
455
* Destroy stored file data if there are no remaining files which reference
456
456
* it.
457
457
*/
458
-
private function deleteFileDataIfUnused(
458
+
public function deleteFileDataIfUnused(
459
459
PhabricatorFileStorageEngine $engine,
460
460
$engine_identifier,
461
461
$handle) {
···
644
644
return $supported;
645
645
}
646
646
647
-
protected function instantiateStorageEngine() {
647
+
public function instantiateStorageEngine() {
648
648
return self::buildEngine($this->getStorageEngine());
649
649
}
650
650