@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add an "--auto" flag to "bin/differential migrate-hunk"
Summary:
Depends on D19370. See T13124. See PHI549. The particular install in PHI549 migrated a large amount of data via the fallback hunk migration script, which does not compress hunks.
Add a mode to `bin/differential migrate-hunk` that amounts to "compress all the hunks which would benefit from compression".
Test Plan: Ran `bin/differential migrate-hunk` with `--auto`, `--all`, `--to`, `--id`, and `--dry-run` in various mixtures. Forced a bunch of hunks to raw ("byte") format, saw it cleanly upgrade them to compressed ("gzde") format.
Reviewers: amckinley
Reviewed By: amckinley
Differential Revision: https://secure.phabricator.com/D19371
+114
-24
+100
-16
src/applications/differential/management/PhabricatorDifferentialMigrateHunkWorkflow.php
+100
-16
src/applications/differential/management/PhabricatorDifferentialMigrateHunkWorkflow.php
···
26
26
'name' => 'all',
27
27
'help' => pht('Migrate all hunks.'),
28
28
),
29
+
array(
30
+
'name' => 'auto',
31
+
'help' => pht('Select storage format automatically.'),
32
+
),
33
+
array(
34
+
'name' => 'dry-run',
35
+
'help' => pht('Show planned writes but do not perform them.'),
36
+
),
29
37
));
30
38
}
31
39
32
40
public function execute(PhutilArgumentParser $args) {
41
+
$is_dry_run = $args->getArg('dry-run');
42
+
33
43
$id = $args->getArg('id');
34
44
$is_all = $args->getArg('all');
35
45
···
45
55
'with "--all".'));
46
56
}
47
57
58
+
$is_auto = $args->getArg('auto');
48
59
$storage = $args->getArg('to');
49
-
switch ($storage) {
50
-
case DifferentialHunk::DATATYPE_TEXT:
51
-
case DifferentialHunk::DATATYPE_FILE:
52
-
break;
53
-
default:
60
+
if ($is_auto && $storage) {
61
+
throw new PhutilArgumentUsageException(
62
+
pht(
63
+
'Options "--to" (to choose a specific storage format) and "--auto" '.
64
+
'(to select a storage format automatically) are mutually '.
65
+
'exclusive.'));
66
+
} else if (!$is_auto && !$storage) {
67
+
throw new PhutilArgumentUsageException(
68
+
pht(
69
+
'Use "--to" to choose a storage format, or "--auto" to select a '.
70
+
'format automatically.'));
71
+
}
72
+
73
+
$types = array(
74
+
DifferentialHunk::DATATYPE_TEXT,
75
+
DifferentialHunk::DATATYPE_FILE,
76
+
);
77
+
$types = array_fuse($types);
78
+
if (strlen($storage)) {
79
+
if (!isset($types[$storage])) {
54
80
throw new PhutilArgumentUsageException(
55
-
pht('Specify a hunk storage engine with --to.'));
81
+
pht(
82
+
'Storage type "%s" is unknown. Supported types are: %s.',
83
+
$storage,
84
+
implode(', ', array_keys($types))));
85
+
}
56
86
}
57
87
58
88
if ($id) {
···
64
94
65
95
foreach ($hunks as $hunk) {
66
96
try {
67
-
$this->migrateHunk($hunk, $storage);
97
+
$this->migrateHunk($hunk, $storage, $is_auto, $is_dry_run);
68
98
} catch (Exception $ex) {
69
99
// If we're migrating a single hunk, just throw the exception. If
70
100
// we're migrating multiple hunks, warn but continue.
···
96
126
return $hunk;
97
127
}
98
128
99
-
private function migrateHunk(DifferentialHunk $hunk, $format) {
129
+
private function migrateHunk(
130
+
DifferentialHunk $hunk,
131
+
$type,
132
+
$is_auto,
133
+
$is_dry_run) {
134
+
135
+
$old_type = $hunk->getDataType();
136
+
137
+
if ($is_auto) {
138
+
// By default, we're just going to keep hunks in the same storage
139
+
// engine. In the future, we could perhaps select large hunks stored in
140
+
// text engine and move them into file storage.
141
+
$new_type = $old_type;
142
+
} else {
143
+
$new_type = $type;
144
+
}
145
+
146
+
// Figure out if the storage format (e.g., plain text vs compressed)
147
+
// would change if we wrote this hunk anew today.
148
+
$old_format = $hunk->getDataFormat();
149
+
$new_format = $hunk->getAutomaticDataFormat();
150
+
151
+
$same_type = ($old_type === $new_type);
152
+
$same_format = ($old_format === $new_format);
153
+
154
+
// If we aren't going to change the storage engine and aren't going to
155
+
// change the storage format, just bail out.
156
+
if ($same_type && $same_format) {
157
+
$this->logInfo(
158
+
pht('SKIP'),
159
+
pht(
160
+
'Hunk %d is already stored in the preferred engine ("%s") '.
161
+
'with the preferred format ("%s").',
162
+
$hunk->getID(),
163
+
$new_type,
164
+
$new_format));
165
+
return;
166
+
}
167
+
168
+
if ($is_dry_run) {
169
+
$this->logOkay(
170
+
pht('DRY RUN'),
171
+
pht(
172
+
'Hunk %d would be rewritten (storage: "%s" -> "%s"; '.
173
+
'format: "%s" -> "%s").',
174
+
$hunk->getID(),
175
+
$old_type,
176
+
$new_type,
177
+
$old_format,
178
+
$new_format));
179
+
return;
180
+
}
181
+
100
182
$old_data = $hunk->getChanges();
101
183
102
-
switch ($format) {
184
+
switch ($new_type) {
103
185
case DifferentialHunk::DATATYPE_TEXT:
104
186
$hunk->saveAsText();
105
-
$this->logOkay(
106
-
pht('TEXT'),
107
-
pht('Converted hunk to text storage.'));
108
187
break;
109
188
case DifferentialHunk::DATATYPE_FILE:
110
189
$hunk->saveAsFile();
111
-
$this->logOkay(
112
-
pht('FILE'),
113
-
pht('Converted hunk to file storage.'));
114
190
break;
115
191
}
116
192
193
+
$this->logOkay(
194
+
pht('MIGRATE'),
195
+
pht(
196
+
'Converted hunk %d to "%s" storage (with format "%s").',
197
+
$hunk->getID(),
198
+
$new_type,
199
+
$hunk->getDataFormat()));
200
+
117
201
$hunk = $this->loadHunk($hunk->getID());
118
202
$new_data = $hunk->getChanges();
119
203
120
204
if ($old_data !== $new_data) {
121
205
throw new Exception(
122
206
pht(
123
-
'Integrity check failed: new file data differs fom old data!'));
207
+
'Integrity check failed: new file data differs from old data!'));
124
208
}
125
209
}
126
210
+14
-8
src/applications/differential/storage/DifferentialHunk.php
+14
-8
src/applications/differential/storage/DifferentialHunk.php
···
290
290
return array(self::DATAFORMAT_RAW, $data);
291
291
}
292
292
293
+
public function getAutomaticDataFormat() {
294
+
// If the hunk is already stored deflated, just keep it deflated. This is
295
+
// mostly a performance improvement for "bin/differential migrate-hunk" so
296
+
// that we don't have to recompress all the stored hunks when looking for
297
+
// stray uncompressed hunks.
298
+
if ($this->dataFormat === self::DATAFORMAT_DEFLATED) {
299
+
return self::DATAFORMAT_DEFLATED;
300
+
}
301
+
302
+
list($format) = $this->formatDataForStorage($this->getRawData());
303
+
304
+
return $format;
305
+
}
306
+
293
307
public function saveAsText() {
294
308
$old_type = $this->getDataType();
295
309
$old_data = $this->getData();
296
-
297
-
if ($old_type == self::DATATYPE_TEXT) {
298
-
return $this;
299
-
}
300
310
301
311
$raw_data = $this->getRawData();
302
312
···
316
326
public function saveAsFile() {
317
327
$old_type = $this->getDataType();
318
328
$old_data = $this->getData();
319
-
320
-
if ($old_type == self::DATATYPE_FILE) {
321
-
return $this;
322
-
}
323
329
324
330
$raw_data = $this->getRawData();
325
331