@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Provide "bin/files integrity" for debugging, maintaining and backfilling integrity hashes
Summary:
Ref T12470. Provides an "integrity" utility which runs in these modes:
- Verify: check that hashes match.
- Compute: backfill missing hashes.
- Strip: remove hashes. Useful for upgrading across a hash change.
- Corrupt: intentionally corrupt hashes. Useful for debugging.
- Overwrite: force hash recomputation.
Users normally shouldn't need to run any of this stuff, but this provides a reasonable toolkit for managing integrity hashes.
I'll recommend existing installs use `bin/files integrity --compute all` in the upgrade guidance to backfill hashes for existing files.
Test Plan:
- Ran the script in many modes against various files, saw expected operation, including:
- Verified a file, corrupted it, saw it fail.
- Verified a file, stripped it, saw it have no hash.
- Stripped a file, computed it, got a clean verify.
- Stripped a file, overwrote it, got a clean verify.
- Corrupted a file, overwrote it, got a clean verify.
- Overwrote a file, overwrote again, got a no-op.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T12470
Differential Revision: https://secure.phabricator.com/D17629
+391
-7
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
2810
2810
'PhabricatorFilesManagementEncodeWorkflow' => 'applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php',
2811
2811
'PhabricatorFilesManagementEnginesWorkflow' => 'applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php',
2812
2812
'PhabricatorFilesManagementGenerateKeyWorkflow' => 'applications/files/management/PhabricatorFilesManagementGenerateKeyWorkflow.php',
2813
+
'PhabricatorFilesManagementIntegrityWorkflow' => 'applications/files/management/PhabricatorFilesManagementIntegrityWorkflow.php',
2813
2814
'PhabricatorFilesManagementMigrateWorkflow' => 'applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php',
2814
2815
'PhabricatorFilesManagementPurgeWorkflow' => 'applications/files/management/PhabricatorFilesManagementPurgeWorkflow.php',
2815
2816
'PhabricatorFilesManagementRebuildWorkflow' => 'applications/files/management/PhabricatorFilesManagementRebuildWorkflow.php',
···
7941
7942
'PhabricatorFilesManagementEncodeWorkflow' => 'PhabricatorFilesManagementWorkflow',
7942
7943
'PhabricatorFilesManagementEnginesWorkflow' => 'PhabricatorFilesManagementWorkflow',
7943
7944
'PhabricatorFilesManagementGenerateKeyWorkflow' => 'PhabricatorFilesManagementWorkflow',
7945
+
'PhabricatorFilesManagementIntegrityWorkflow' => 'PhabricatorFilesManagementWorkflow',
7944
7946
'PhabricatorFilesManagementMigrateWorkflow' => 'PhabricatorFilesManagementWorkflow',
7945
7947
'PhabricatorFilesManagementPurgeWorkflow' => 'PhabricatorFilesManagementWorkflow',
7946
7948
'PhabricatorFilesManagementRebuildWorkflow' => 'PhabricatorFilesManagementWorkflow',
+1
-1
src/applications/files/engine/PhabricatorFileStorageEngine.php
+1
-1
src/applications/files/engine/PhabricatorFileStorageEngine.php
···
336
336
$known_integrity = $file->getIntegrityHash();
337
337
if ($known_integrity !== null) {
338
338
$new_integrity = $this->newIntegrityHash($formatted_data, $format);
339
-
if ($known_integrity !== $new_integrity) {
339
+
if (!phutil_hashes_are_identical($known_integrity, $new_integrity)) {
340
340
throw new PhabricatorFileIntegrityException(
341
341
pht(
342
342
'File data integrity check failed. Dark forces have corrupted '.
+325
src/applications/files/management/PhabricatorFilesManagementIntegrityWorkflow.php
+325
src/applications/files/management/PhabricatorFilesManagementIntegrityWorkflow.php
···
1
+
<?php
2
+
3
+
final class PhabricatorFilesManagementIntegrityWorkflow
4
+
extends PhabricatorFilesManagementWorkflow {
5
+
6
+
protected function didConstruct() {
7
+
$this
8
+
->setName('integrity')
9
+
->setSynopsis(pht('Verify or recalculate file integrity hashes.'))
10
+
->setArguments(
11
+
array(
12
+
array(
13
+
'name' => 'all',
14
+
'help' => pht('Affect all files.'),
15
+
),
16
+
array(
17
+
'name' => 'strip',
18
+
'help' => pht(
19
+
'DANGEROUS. Strip integrity hashes from files. This makes '.
20
+
'files vulnerable to corruption or tampering.'),
21
+
),
22
+
array(
23
+
'name' => 'corrupt',
24
+
'help' => pht(
25
+
'Corrupt integrity hashes for given files. This is intended '.
26
+
'for debugging.'),
27
+
),
28
+
array(
29
+
'name' => 'compute',
30
+
'help' => pht(
31
+
'Compute and update integrity hashes for files which do not '.
32
+
'yet have them.'),
33
+
),
34
+
array(
35
+
'name' => 'overwrite',
36
+
'help' => pht(
37
+
'DANGEROUS. Recompute and update integrity hashes, overwriting '.
38
+
'invalid hashes. This may mark corrupt or dangerous files as '.
39
+
'valid.'),
40
+
),
41
+
array(
42
+
'name' => 'force',
43
+
'short' => 'f',
44
+
'help' => pht(
45
+
'Execute dangerous operations without prompting for '.
46
+
'confirmation.'),
47
+
),
48
+
array(
49
+
'name' => 'names',
50
+
'wildcard' => true,
51
+
),
52
+
));
53
+
}
54
+
55
+
public function execute(PhutilArgumentParser $args) {
56
+
$modes = array();
57
+
58
+
$is_strip = $args->getArg('strip');
59
+
if ($is_strip) {
60
+
$modes[] = 'strip';
61
+
}
62
+
63
+
$is_corrupt = $args->getArg('corrupt');
64
+
if ($is_corrupt) {
65
+
$modes[] = 'corrupt';
66
+
}
67
+
68
+
$is_compute = $args->getArg('compute');
69
+
if ($is_compute) {
70
+
$modes[] = 'compute';
71
+
}
72
+
73
+
$is_overwrite = $args->getArg('overwrite');
74
+
if ($is_overwrite) {
75
+
$modes[] = 'overwrite';
76
+
}
77
+
78
+
$is_verify = !$modes;
79
+
if ($is_verify) {
80
+
$modes[] = 'verify';
81
+
}
82
+
83
+
if (count($modes) > 1) {
84
+
throw new PhutilArgumentUsageException(
85
+
pht(
86
+
'You have selected multiple operation modes (%s). Choose a '.
87
+
'single mode to operate in.',
88
+
implode(', ', $modes)));
89
+
}
90
+
91
+
$is_force = $args->getArg('force');
92
+
if (!$is_force) {
93
+
$prompt = null;
94
+
if ($is_strip) {
95
+
$prompt = pht(
96
+
'Stripping integrity hashes is dangerous and makes files '.
97
+
'vulnerable to corruption or tampering.');
98
+
}
99
+
100
+
if ($is_corrupt) {
101
+
$prompt = pht(
102
+
'Corrupting integrity hashes will prevent files from being '.
103
+
'accessed. This mode is intended only for development and '.
104
+
'debugging.');
105
+
}
106
+
107
+
if ($is_overwrite) {
108
+
$prompt = pht(
109
+
'Overwriting integrity hashes is dangerous and may mark files '.
110
+
'which have been corrupted or tampered with as safe.');
111
+
}
112
+
113
+
if ($prompt) {
114
+
$this->logWarn(pht('DANGEROUS'), $prompt);
115
+
116
+
if (!phutil_console_confirm(pht('Continue anyway?'))) {
117
+
throw new PhutilArgumentUsageException(pht('Aborted workflow.'));
118
+
}
119
+
}
120
+
}
121
+
122
+
$iterator = $this->buildIterator($args);
123
+
if (!$iterator) {
124
+
throw new PhutilArgumentUsageException(
125
+
pht(
126
+
'Either specify a list of files to affect, or use "--all" to '.
127
+
'affect all files.'));
128
+
}
129
+
130
+
$failure_count = 0;
131
+
$total_count = 0;
132
+
133
+
foreach ($iterator as $file) {
134
+
$total_count++;
135
+
$display_name = $file->getMonogram();
136
+
137
+
$old_hash = $file->getIntegrityHash();
138
+
139
+
if ($is_strip) {
140
+
if ($old_hash === null) {
141
+
$this->logInfo(
142
+
pht('SKIPPED'),
143
+
pht(
144
+
'File "%s" does not have an integrity hash to strip.',
145
+
$display_name));
146
+
} else {
147
+
$file
148
+
->setIntegrityHash(null)
149
+
->save();
150
+
151
+
$this->logWarn(
152
+
pht('STRIPPED'),
153
+
pht(
154
+
'Stripped integrity hash for "%s".',
155
+
$display_name));
156
+
}
157
+
158
+
continue;
159
+
}
160
+
161
+
$need_hash = ($is_verify && $old_hash) ||
162
+
($is_compute && ($old_hash === null)) ||
163
+
($is_corrupt) ||
164
+
($is_overwrite);
165
+
if ($need_hash) {
166
+
try {
167
+
$new_hash = $file->newIntegrityHash();
168
+
} catch (Exception $ex) {
169
+
$failure_count++;
170
+
171
+
$this->logFail(
172
+
pht('ERROR'),
173
+
pht(
174
+
'Unable to compute integrity hash for file "%s": %s',
175
+
$display_name,
176
+
$ex->getMessage()));
177
+
178
+
continue;
179
+
}
180
+
} else {
181
+
$new_hash = null;
182
+
}
183
+
184
+
// NOTE: When running in "corrupt" mode, we only corrupt the hash if
185
+
// we're able to compute a valid hash. Some files, like chunked files,
186
+
// do not support integrity hashing so corrupting them would create an
187
+
// unusual state.
188
+
189
+
if ($is_corrupt) {
190
+
if ($new_hash === null) {
191
+
$this->logInfo(
192
+
pht('IGNORED'),
193
+
pht(
194
+
'Storage for file "%s" does not support integrity hashing.',
195
+
$display_name));
196
+
} else {
197
+
$file
198
+
->setIntegrityHash('<corrupted>')
199
+
->save();
200
+
201
+
$this->logWarn(
202
+
pht('CORRUPTED'),
203
+
pht(
204
+
'Corrupted integrity hash for file "%s".',
205
+
$display_name));
206
+
}
207
+
208
+
continue;
209
+
}
210
+
211
+
if ($is_verify) {
212
+
if ($old_hash === null) {
213
+
$this->logInfo(
214
+
pht('NONE'),
215
+
pht(
216
+
'File "%s" has no stored integrity hash.',
217
+
$display_name));
218
+
} else if ($new_hash === null) {
219
+
$failure_count++;
220
+
221
+
$this->logWarn(
222
+
pht('UNEXPECTED'),
223
+
pht(
224
+
'Storage for file "%s" does not support integrity hashing, '.
225
+
'but the file has an integrity hash.',
226
+
$display_name));
227
+
} else if (phutil_hashes_are_identical($old_hash, $new_hash)) {
228
+
$this->logOkay(
229
+
pht('VALID'),
230
+
pht(
231
+
'File "%s" has a valid integrity hash.',
232
+
$display_name));
233
+
} else {
234
+
$failure_count++;
235
+
236
+
$this->logFail(
237
+
pht('MISMATCH'),
238
+
pht(
239
+
'File "%s" has an invalid integrity hash!',
240
+
$display_name));
241
+
}
242
+
243
+
continue;
244
+
}
245
+
246
+
if ($is_compute) {
247
+
if ($old_hash !== null) {
248
+
$this->logInfo(
249
+
pht('SKIP'),
250
+
pht(
251
+
'File "%s" already has an integrity hash.',
252
+
$display_name));
253
+
} else if ($new_hash === null) {
254
+
$this->logInfo(
255
+
pht('IGNORED'),
256
+
pht(
257
+
'Storage for file "%s" does not support integrity hashing.',
258
+
$display_name));
259
+
} else {
260
+
$file
261
+
->setIntegrityHash($new_hash)
262
+
->save();
263
+
264
+
$this->logOkay(
265
+
pht('COMPUTE'),
266
+
pht(
267
+
'Computed and stored integrity hash for file "%s".',
268
+
$display_name));
269
+
}
270
+
271
+
continue;
272
+
}
273
+
274
+
if ($is_overwrite) {
275
+
$same_hash = ($old_hash !== null) &&
276
+
($new_hash !== null) &&
277
+
phutil_hashes_are_identical($old_hash, $new_hash);
278
+
279
+
if ($new_hash === null) {
280
+
$this->logInfo(
281
+
pht('IGNORED'),
282
+
pht(
283
+
'Storage for file "%s" does not support integrity hashing.',
284
+
$display_name));
285
+
} else if ($same_hash) {
286
+
$this->logInfo(
287
+
pht('UNCHANGED'),
288
+
pht(
289
+
'File "%s" already has the correct integrity hash.',
290
+
$display_name));
291
+
} else {
292
+
$file
293
+
->setIntegrityHash($new_hash)
294
+
->save();
295
+
296
+
$this->logOkay(
297
+
pht('OVERWRITE'),
298
+
pht(
299
+
'Overwrote integrity hash for file "%s".',
300
+
$display_name));
301
+
}
302
+
303
+
continue;
304
+
}
305
+
}
306
+
307
+
if ($failure_count) {
308
+
$this->logFail(
309
+
pht('FAIL'),
310
+
pht(
311
+
'Processed %s file(s), encountered %s error(s).',
312
+
new PhutilNumber($total_count),
313
+
new PhutilNumber($failure_count)));
314
+
} else {
315
+
$this->logOkay(
316
+
pht('DONE'),
317
+
pht(
318
+
'Processed %s file(s) with no errors.',
319
+
new PhutilNumber($total_count)));
320
+
}
321
+
322
+
return 0;
323
+
}
324
+
325
+
}
+27
-6
src/applications/files/storage/PhabricatorFile.php
+27
-6
src/applications/files/storage/PhabricatorFile.php
···
493
493
494
494
$engine_class = get_class($engine);
495
495
496
-
$key = $this->getStorageFormat();
497
-
$format = id(clone PhabricatorFileStorageFormat::requireFormat($key))
498
-
->setFile($this);
496
+
$format = $this->newStorageFormat();
499
497
500
498
$data_iterator = array($data);
501
499
$formatted_iterator = $format->newWriteIterator($data_iterator);
···
756
754
public function getFileDataIterator($begin = null, $end = null) {
757
755
$engine = $this->instantiateStorageEngine();
758
756
759
-
$key = $this->getStorageFormat();
760
-
$format = id(clone PhabricatorFileStorageFormat::requireFormat($key))
761
-
->setFile($this);
757
+
$format = $this->newStorageFormat();
762
758
763
759
$iterator = $engine->getRawFileDataIterator(
764
760
$this,
···
1238
1234
return idx($this->metadata, self::METADATA_INTEGRITY);
1239
1235
}
1240
1236
1237
+
public function newIntegrityHash() {
1238
+
$engine = $this->instantiateStorageEngine();
1239
+
1240
+
if ($engine->isChunkEngine()) {
1241
+
return null;
1242
+
}
1243
+
1244
+
$format = $this->newStorageFormat();
1245
+
1246
+
$storage_handle = $this->getStorageHandle();
1247
+
$data = $engine->readFile($storage_handle);
1248
+
1249
+
return $engine->newIntegrityHash($data, $format);
1250
+
}
1251
+
1241
1252
/**
1242
1253
* Write the policy edge between this file and some object.
1243
1254
*
···
1404
1415
1405
1416
public function getTransform($key) {
1406
1417
return $this->assertAttachedKey($this->transforms, $key);
1418
+
}
1419
+
1420
+
public function newStorageFormat() {
1421
+
$key = $this->getStorageFormat();
1422
+
$template = PhabricatorFileStorageFormat::requireFormat($key);
1423
+
1424
+
$format = id(clone $template)
1425
+
->setFile($this);
1426
+
1427
+
return $format;
1407
1428
}
1408
1429
1409
1430
+36
src/infrastructure/management/PhabricatorManagementWorkflow.php
+36
src/infrastructure/management/PhabricatorManagementWorkflow.php
···
31
31
PhabricatorConsoleContentSource::SOURCECONST);
32
32
}
33
33
34
+
protected function logInfo($label, $message) {
35
+
$this->logRaw(
36
+
tsprintf(
37
+
"**<bg:blue> %s </bg>** %s\n",
38
+
$label,
39
+
$message));
40
+
}
41
+
42
+
protected function logOkay($label, $message) {
43
+
$this->logRaw(
44
+
tsprintf(
45
+
"**<bg:green> %s </bg>** %s\n",
46
+
$label,
47
+
$message));
48
+
}
49
+
50
+
protected function logWarn($label, $message) {
51
+
$this->logRaw(
52
+
tsprintf(
53
+
"**<bg:yellow> %s </bg>** %s\n",
54
+
$label,
55
+
$message));
56
+
}
57
+
58
+
protected function logFail($label, $message) {
59
+
$this->logRaw(
60
+
tsprintf(
61
+
"**<bg:red> %s </bg>** %s\n",
62
+
$label,
63
+
$message));
64
+
}
65
+
66
+
private function logRaw($message) {
67
+
fprintf(STDERR, '%s', $message);
68
+
}
69
+
34
70
}