@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Support storage of Differential hunk data in Files
Summary:
Ref T12932. For long-lived installs, one of the largest tables tends to be the hunk data table. Although it doesn't grow tremendously fast, it's also well suited to storage in Files instead of the database (infrequent access, relatively large blobs of data, mostly one-at-a-time access), and earlier work anticipated eventually adding support for Files storage.
Make Files storage work, and provide `bin/differential migrate-hunk` to manually test/migrate hunks. This is currently the only way hunks get moved to file storage, but I expect to add a GC step which moves them to File storage after 30 days shortly.
The immediate motivation for this is to relieve storage pressure on db001/db002 so we have more headroom for deploying the Ferret engine and its larger indexes (see also T12819).
Test Plan:
- Used `bin/differential migrate-hunk` to move a hunk to and from file storage, verified it survived intact.
- Downloaded the actual stored file, sanity-checked it. Verified permissions.
- Destroyed a diff with `bin/remove destroy`, saw the hunk and file storage destroyed.
- Verified that going from file -> text destroys the old file properly with `migrate-hunk --trace ...`.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T12932
Differential Revision: https://secure.phabricator.com/D18584
+257
-5
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
2667
2667
'PhabricatorDifferentialConfigOptions' => 'applications/differential/config/PhabricatorDifferentialConfigOptions.php',
2668
2668
'PhabricatorDifferentialExtractWorkflow' => 'applications/differential/management/PhabricatorDifferentialExtractWorkflow.php',
2669
2669
'PhabricatorDifferentialManagementWorkflow' => 'applications/differential/management/PhabricatorDifferentialManagementWorkflow.php',
2670
+
'PhabricatorDifferentialMigrateHunkWorkflow' => 'applications/differential/management/PhabricatorDifferentialMigrateHunkWorkflow.php',
2670
2671
'PhabricatorDifferentialRevisionTestDataGenerator' => 'applications/differential/lipsum/PhabricatorDifferentialRevisionTestDataGenerator.php',
2671
2672
'PhabricatorDiffusionApplication' => 'applications/diffusion/application/PhabricatorDiffusionApplication.php',
2672
2673
'PhabricatorDiffusionBlameSetting' => 'applications/settings/setting/PhabricatorDiffusionBlameSetting.php',
···
5367
5368
'DifferentialChangeset' => array(
5368
5369
'DifferentialDAO',
5369
5370
'PhabricatorPolicyInterface',
5371
+
'PhabricatorDestructibleInterface',
5370
5372
),
5371
5373
'DifferentialChangesetDetailView' => 'AphrontView',
5372
5374
'DifferentialChangesetFileTreeSideNavBuilder' => 'Phobject',
···
5461
5463
'DifferentialHunk' => array(
5462
5464
'DifferentialDAO',
5463
5465
'PhabricatorPolicyInterface',
5466
+
'PhabricatorDestructibleInterface',
5464
5467
),
5465
5468
'DifferentialHunkParser' => 'Phobject',
5466
5469
'DifferentialHunkParserTestCase' => 'PhabricatorTestCase',
···
7999
8002
'PhabricatorDifferentialConfigOptions' => 'PhabricatorApplicationConfigOptions',
8000
8003
'PhabricatorDifferentialExtractWorkflow' => 'PhabricatorDifferentialManagementWorkflow',
8001
8004
'PhabricatorDifferentialManagementWorkflow' => 'PhabricatorManagementWorkflow',
8005
+
'PhabricatorDifferentialMigrateHunkWorkflow' => 'PhabricatorDifferentialManagementWorkflow',
8002
8006
'PhabricatorDifferentialRevisionTestDataGenerator' => 'PhabricatorTestDataGenerator',
8003
8007
'PhabricatorDiffusionApplication' => 'PhabricatorApplication',
8004
8008
'PhabricatorDiffusionBlameSetting' => 'PhabricatorInternalSetting',
+86
src/applications/differential/management/PhabricatorDifferentialMigrateHunkWorkflow.php
+86
src/applications/differential/management/PhabricatorDifferentialMigrateHunkWorkflow.php
···
1
+
<?php
2
+
3
+
final class PhabricatorDifferentialMigrateHunkWorkflow
4
+
extends PhabricatorDifferentialManagementWorkflow {
5
+
6
+
protected function didConstruct() {
7
+
$this
8
+
->setName('migrate-hunk')
9
+
->setExamples('**migrate-hunk** --id __hunk__ --to __storage__')
10
+
->setSynopsis(pht('Migrate storage engines for a hunk.'))
11
+
->setArguments(
12
+
array(
13
+
array(
14
+
'name' => 'id',
15
+
'param' => 'id',
16
+
'help' => pht('Hunk ID to migrate.'),
17
+
),
18
+
array(
19
+
'name' => 'to',
20
+
'param' => 'storage',
21
+
'help' => pht('Storage engine to migrate to.'),
22
+
),
23
+
));
24
+
}
25
+
26
+
public function execute(PhutilArgumentParser $args) {
27
+
$id = $args->getArg('id');
28
+
if (!$id) {
29
+
throw new PhutilArgumentUsageException(
30
+
pht('Specify a hunk to migrate with --id.'));
31
+
}
32
+
33
+
$storage = $args->getArg('to');
34
+
switch ($storage) {
35
+
case DifferentialModernHunk::DATATYPE_TEXT:
36
+
case DifferentialModernHunk::DATATYPE_FILE:
37
+
break;
38
+
default:
39
+
throw new PhutilArgumentUsageException(
40
+
pht('Specify a hunk storage engine with --to.'));
41
+
}
42
+
43
+
$hunk = $this->loadHunk($id);
44
+
$old_data = $hunk->getChanges();
45
+
46
+
switch ($storage) {
47
+
case DifferentialModernHunk::DATATYPE_TEXT:
48
+
$hunk->saveAsText();
49
+
$this->logOkay(
50
+
pht('TEXT'),
51
+
pht('Convereted hunk to text storage.'));
52
+
break;
53
+
case DifferentialModernHunk::DATATYPE_FILE:
54
+
$hunk->saveAsFile();
55
+
$this->logOkay(
56
+
pht('FILE'),
57
+
pht('Convereted hunk to file storage.'));
58
+
break;
59
+
}
60
+
61
+
$hunk = $this->loadHunk($id);
62
+
$new_data = $hunk->getChanges();
63
+
64
+
if ($old_data !== $new_data) {
65
+
throw new Exception(
66
+
pht(
67
+
'Integrity check failed: new file data differs fom old data!'));
68
+
}
69
+
70
+
return 0;
71
+
}
72
+
73
+
private function loadHunk($id) {
74
+
$hunk = id(new DifferentialModernHunk())->load($id);
75
+
if (!$hunk) {
76
+
throw new PhutilArgumentUsageException(
77
+
pht(
78
+
'No hunk exists with ID "%s".',
79
+
$id));
80
+
}
81
+
82
+
return $hunk;
83
+
}
84
+
85
+
86
+
}
+26
-2
src/applications/differential/storage/DifferentialChangeset.php
+26
-2
src/applications/differential/storage/DifferentialChangeset.php
···
1
1
<?php
2
2
3
-
final class DifferentialChangeset extends DifferentialDAO
4
-
implements PhabricatorPolicyInterface {
3
+
final class DifferentialChangeset
4
+
extends DifferentialDAO
5
+
implements
6
+
PhabricatorPolicyInterface,
7
+
PhabricatorDestructibleInterface {
5
8
6
9
protected $diffID;
7
10
protected $oldFile;
···
235
238
public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
236
239
return $this->getDiff()->hasAutomaticCapability($capability, $viewer);
237
240
}
241
+
242
+
243
+
/* -( PhabricatorDestructibleInterface )----------------------------------- */
244
+
245
+
246
+
public function destroyObjectPermanently(
247
+
PhabricatorDestructionEngine $engine) {
248
+
$this->openTransaction();
249
+
250
+
$hunks = id(new DifferentialModernHunk())->loadAllWhere(
251
+
'changesetID = %d',
252
+
$this->getID());
253
+
foreach ($hunks as $hunk) {
254
+
$engine->destroyObject($hunk);
255
+
}
256
+
257
+
$this->delete();
258
+
259
+
$this->saveTransaction();
260
+
}
261
+
238
262
239
263
}
+1
-1
src/applications/differential/storage/DifferentialDiff.php
+1
-1
src/applications/differential/storage/DifferentialDiff.php
+15
-2
src/applications/differential/storage/DifferentialHunk.php
+15
-2
src/applications/differential/storage/DifferentialHunk.php
···
1
1
<?php
2
2
3
-
abstract class DifferentialHunk extends DifferentialDAO
4
-
implements PhabricatorPolicyInterface {
3
+
abstract class DifferentialHunk
4
+
extends DifferentialDAO
5
+
implements
6
+
PhabricatorPolicyInterface,
7
+
PhabricatorDestructibleInterface {
5
8
6
9
protected $changesetID;
7
10
protected $oldOffset;
···
227
230
public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
228
231
return $this->getChangeset()->hasAutomaticCapability($capability, $viewer);
229
232
}
233
+
234
+
235
+
/* -( PhabricatorDestructibleInterface )----------------------------------- */
236
+
237
+
238
+
public function destroyObjectPermanently(
239
+
PhabricatorDestructionEngine $engine) {
240
+
$this->delete();
241
+
}
242
+
230
243
231
244
}
+125
src/applications/differential/storage/DifferentialModernHunk.php
+125
src/applications/differential/storage/DifferentialModernHunk.php
···
15
15
16
16
private $rawData;
17
17
private $forcedEncoding;
18
+
private $fileData;
18
19
19
20
public function getTableName() {
20
21
return 'differential_hunk_modern';
···
87
88
return parent::save();
88
89
}
89
90
91
+
public function saveAsText() {
92
+
$old_type = $this->getDataType();
93
+
$old_data = $this->getData();
94
+
95
+
if ($old_type == self::DATATYPE_TEXT) {
96
+
return $this;
97
+
}
98
+
99
+
$raw_data = $this->getRawData();
100
+
101
+
$this->setDataType(self::DATATYPE_TEXT);
102
+
$this->setData($raw_data);
103
+
104
+
$result = $this->save();
105
+
106
+
$this->destroyData($old_type, $old_data);
107
+
108
+
return $result;
109
+
}
110
+
111
+
public function saveAsFile() {
112
+
$old_type = $this->getDataType();
113
+
$old_data = $this->getData();
114
+
115
+
if ($old_type == self::DATATYPE_FILE) {
116
+
return $this;
117
+
}
118
+
119
+
$raw_data = $this->getRawData();
120
+
121
+
$file = PhabricatorFile::newFromFileData(
122
+
$raw_data,
123
+
array(
124
+
'name' => 'differential-hunk',
125
+
'mime-type' => 'application/octet-stream',
126
+
'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
127
+
));
128
+
129
+
$this->setDataType(self::DATATYPE_FILE);
130
+
$this->setData($file->getPHID());
131
+
132
+
// NOTE: Because hunks don't have a PHID and we just load hunk data with
133
+
// the ominipotent viewer, we do not need to attach the file to anything.
134
+
135
+
$result = $this->save();
136
+
137
+
$this->destroyData($old_type, $old_data);
138
+
139
+
return $result;
140
+
}
141
+
90
142
private function getRawData() {
91
143
if ($this->rawData === null) {
92
144
$type = $this->getDataType();
···
98
150
$data = $data;
99
151
break;
100
152
case self::DATATYPE_FILE:
153
+
$data = $this->loadFileData();
154
+
break;
101
155
default:
102
156
throw new Exception(
103
157
pht('Hunk has unsupported data type "%s"!', $type));
···
121
175
}
122
176
123
177
return $this->rawData;
178
+
}
179
+
180
+
private function loadFileData() {
181
+
if ($this->fileData === null) {
182
+
$type = $this->getDataType();
183
+
if ($type !== self::DATATYPE_FILE) {
184
+
throw new Exception(
185
+
pht(
186
+
'Unable to load file data for hunk with wrong data type ("%s").',
187
+
$type));
188
+
}
189
+
190
+
$file_phid = $this->getData();
191
+
192
+
$file = $this->loadRawFile($file_phid);
193
+
$data = $file->loadFileData();
194
+
195
+
$this->fileData = $data;
196
+
}
197
+
198
+
return $this->fileData;
199
+
}
200
+
201
+
private function loadRawFile($file_phid) {
202
+
$viewer = PhabricatorUser::getOmnipotentUser();
203
+
204
+
205
+
$files = id(new PhabricatorFileQuery())
206
+
->setViewer($viewer)
207
+
->withPHIDs(array($file_phid))
208
+
->execute();
209
+
if (!$files) {
210
+
throw new Exception(
211
+
pht(
212
+
'Failed to load file ("%s") with hunk data.',
213
+
$file_phid));
214
+
}
215
+
216
+
$file = head($files);
217
+
218
+
return $file;
219
+
}
220
+
221
+
222
+
public function destroyObjectPermanently(
223
+
PhabricatorDestructionEngine $engine) {
224
+
225
+
$type = $this->getDataType();
226
+
$data = $this->getData();
227
+
228
+
$this->destroyData($type, $data, $engine);
229
+
230
+
return parent::destroyObjectPermanently($engine);
231
+
}
232
+
233
+
234
+
private function destroyData(
235
+
$type,
236
+
$data,
237
+
PhabricatorDestructionEngine $engine = null) {
238
+
239
+
if (!$engine) {
240
+
$engine = new PhabricatorDestructionEngine();
241
+
}
242
+
243
+
switch ($type) {
244
+
case self::DATATYPE_FILE:
245
+
$file = $this->loadRawFile($data);
246
+
$engine->destroyObject($file);
247
+
break;
248
+
}
124
249
}
125
250
126
251
}