@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Provide "--output" flags for "bin/storage renamespace"
Summary:
Ref T6996. Depends on D16407. This does the same stuff as D16407, but for `bin/storage renamespace`. In particular:
- Support writing directly to a file (so we can get good errors on failure).
- Support in-process compression.
Also add support for reading out of a `storage dump` subprocess, so we don't have to do a dump-to-disk + renamespace + compress dance and can just stream out of MySQL directly to a compressed file on disk.
This is used in the second stage of instance exports (see T7148).
It would be nice to share more code with `bin/storage dump`, and possibly to just make this a flag for it, although we still do need to do the file-based version when importing (vs exporting). I figured that was better left for another time.
Test Plan:
Ran `bin/storage renamespace --live --output x --from A --to B --compress --overwrite` and similar commands.
Verified that a compressed, renamespaced dump came out of the other end.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T6996
Differential Revision: https://secure.phabricator.com/D16410
+137
-21
+137
-21
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementRenamespaceWorkflow.php
+137
-21
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementRenamespaceWorkflow.php
···
8
8
->setName('renamespace')
9
9
->setExamples(
10
10
'**renamespace** [__options__] '.
11
-
'--in __dump.sql__ --from __old__ --to __new__ > __out.sql__')
11
+
'--input __dump.sql__ --from __old__ --to __new__ > __out.sql__')
12
12
->setSynopsis(pht('Change the database namespace of a .sql dump file.'))
13
13
->setArguments(
14
14
array(
15
15
array(
16
-
'name' => 'in',
16
+
'name' => 'input',
17
17
'param' => 'file',
18
18
'help' => pht('SQL dumpfile to process.'),
19
19
),
20
20
array(
21
+
'name' => 'live',
22
+
'help' => pht(
23
+
'Generate a live dump instead of processing a file on disk.'),
24
+
),
25
+
array(
21
26
'name' => 'from',
22
27
'param' => 'namespace',
23
28
'help' => pht('Current database namespace used by dumpfile.'),
···
27
32
'param' => 'namespace',
28
33
'help' => pht('Desired database namespace for output.'),
29
34
),
35
+
array(
36
+
'name' => 'output',
37
+
'param' => 'file',
38
+
'help' => pht('Write output directly to a file on disk.'),
39
+
),
40
+
array(
41
+
'name' => 'compress',
42
+
'help' => pht('Emit gzipped output instead of plain text.'),
43
+
),
44
+
array(
45
+
'name' => 'overwrite',
46
+
'help' => pht(
47
+
'With __--output__, write to disk even if the file already '.
48
+
'exists.'),
49
+
),
30
50
));
31
51
}
32
52
···
37
57
public function didExecute(PhutilArgumentParser $args) {
38
58
$console = PhutilConsole::getConsole();
39
59
40
-
$in = $args->getArg('in');
41
-
if (!strlen($in)) {
60
+
$input = $args->getArg('input');
61
+
$is_live = $args->getArg('live');
62
+
if (!strlen($input) && !$is_live) {
42
63
throw new PhutilArgumentUsageException(
43
64
pht(
44
-
'Specify the dumpfile to read with %s.',
45
-
'--in'));
65
+
'Specify the dumpfile to read with "--in", or use "--live" to '.
66
+
'generate one automatically.'));
46
67
}
47
68
48
69
$from = $args->getArg('from');
···
61
82
'--to'));
62
83
}
63
84
85
+
86
+
$output_file = $args->getArg('output');
87
+
$is_overwrite = $args->getArg('overwrite');
88
+
$is_compress = $args->getArg('compress');
89
+
90
+
if ($is_overwrite) {
91
+
if ($output_file === null) {
92
+
throw new PhutilArgumentUsageException(
93
+
pht(
94
+
'The "--overwrite" flag can only be used alongside "--output".'));
95
+
}
96
+
}
97
+
98
+
if ($output_file !== null) {
99
+
if (Filesystem::pathExists($output_file)) {
100
+
if (!$is_overwrite) {
101
+
throw new PhutilArgumentUsageException(
102
+
pht(
103
+
'Output file "%s" already exists. Use "--overwrite" '.
104
+
'to overwrite.',
105
+
$output_file));
106
+
}
107
+
}
108
+
}
109
+
110
+
if ($is_live) {
111
+
$root = dirname(phutil_get_library_root('phabricator'));
112
+
113
+
$future = new ExecFuture(
114
+
'%R dump',
115
+
$root.'/bin/storage');
116
+
117
+
$lines = new LinesOfALargeExecFuture($future);
118
+
} else {
119
+
$lines = new LinesOfALargeFile($input);
120
+
}
121
+
122
+
if ($output_file === null) {
123
+
$file = fopen('php://stdout', 'wb');
124
+
$output_name = pht('stdout');
125
+
} else {
126
+
if ($is_compress) {
127
+
$file = gzopen($output_file, 'wb');
128
+
} else {
129
+
$file = fopen($output_file, 'wb');
130
+
}
131
+
$output_name = $output_file;
132
+
}
133
+
134
+
if (!$file) {
135
+
throw new Exception(
136
+
pht(
137
+
'Failed to open output file "%s" for writing.',
138
+
$output_name));
139
+
}
140
+
64
141
$patterns = array(
65
142
'use' => '@^(USE `)([^_]+)(_.*)$@',
66
143
'create' => '@^(CREATE DATABASE /\*.*?\*/ `)([^_]+)(_.*)$@',
···
68
145
69
146
$found = array_fill_keys(array_keys($patterns), 0);
70
147
71
-
$matches = null;
72
-
foreach (new LinesOfALargeFile($in) as $line) {
148
+
try {
149
+
$matches = null;
150
+
foreach ($lines as $line) {
73
151
74
-
foreach ($patterns as $key => $pattern) {
75
-
if (preg_match($pattern, $line, $matches)) {
76
-
$namespace = $matches[2];
77
-
if ($namespace != $from) {
78
-
throw new Exception(
79
-
pht(
80
-
'Expected namespace "%s", found "%s": %s.',
81
-
$from,
82
-
$namespace,
83
-
$line));
152
+
foreach ($patterns as $key => $pattern) {
153
+
if (preg_match($pattern, $line, $matches)) {
154
+
$namespace = $matches[2];
155
+
if ($namespace != $from) {
156
+
throw new Exception(
157
+
pht(
158
+
'Expected namespace "%s", found "%s": %s.',
159
+
$from,
160
+
$namespace,
161
+
$line));
162
+
}
163
+
164
+
$line = $matches[1].$to.$matches[3];
165
+
$found[$key]++;
84
166
}
167
+
}
85
168
86
-
$line = $matches[1].$to.$matches[3];
87
-
$found[$key]++;
169
+
$data = $line."\n";
170
+
171
+
if ($is_compress) {
172
+
$bytes = gzwrite($file, $data);
173
+
} else {
174
+
$bytes = fwrite($file, $data);
175
+
}
176
+
177
+
if ($bytes !== strlen($data)) {
178
+
throw new Exception(
179
+
pht(
180
+
'Failed to write %d byte(s) to "%s".',
181
+
new PhutilNumber(strlen($data)),
182
+
$output_name));
183
+
}
184
+
}
185
+
186
+
if ($is_compress) {
187
+
$ok = gzclose($file);
188
+
} else {
189
+
$ok = fclose($file);
190
+
}
191
+
192
+
if ($ok !== true) {
193
+
throw new Exception(
194
+
pht(
195
+
'Failed to close file "%s".',
196
+
$output_file));
197
+
}
198
+
} catch (Exception $ex) {
199
+
try {
200
+
if ($output_file !== null) {
201
+
Filesystem::remove($output_file);
88
202
}
203
+
} catch (Exception $ex) {
204
+
// Ignore any exception.
89
205
}
90
206
91
-
echo $line."\n";
207
+
throw $ex;
92
208
}
93
209
94
210
// Give the user a chance to catch things if the results are crazy.