@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add a script to migrate files between storage engines
Summary: Quora requested this (moving to S3) but it's also clearly a good idea.
Test Plan:
Ran with various valid/invalid options to test options. Error/sanity checking seemed OK.
Migrated individual local files.
Migrated all my local files back and forth between engines several times.
Uploaded some new files.
Reviewers: btrahan, vrana
Reviewed By: vrana
CC: aran
Maniphest Tasks: T1950
Differential Revision: https://secure.phabricator.com/D3808
+372
-27
+1
bin/files
+1
bin/files
···
1
+
../scripts/files/manage_files.php
+39
scripts/files/manage_files.php
+39
scripts/files/manage_files.php
···
1
+
#!/usr/bin/env php
2
+
<?php
3
+
4
+
/*
5
+
* Copyright 2012 Facebook, Inc.
6
+
*
7
+
* Licensed under the Apache License, Version 2.0 (the "License");
8
+
* you may not use this file except in compliance with the License.
9
+
* You may obtain a copy of the License at
10
+
*
11
+
* http://www.apache.org/licenses/LICENSE-2.0
12
+
*
13
+
* Unless required by applicable law or agreed to in writing, software
14
+
* distributed under the License is distributed on an "AS IS" BASIS,
15
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+
* See the License for the specific language governing permissions and
17
+
* limitations under the License.
18
+
*/
19
+
20
+
$root = dirname(dirname(dirname(__FILE__)));
21
+
require_once $root.'/scripts/__init_script__.php';
22
+
23
+
$args = new PhutilArgumentParser($argv);
24
+
$args->setTagline('manage files');
25
+
$args->setSynopsis(<<<EOSYNOPSIS
26
+
**files** __command__ [__options__]
27
+
Manage Phabricator file storage.
28
+
29
+
EOSYNOPSIS
30
+
);
31
+
$args->parseStandardArguments();
32
+
33
+
$workflows = array(
34
+
new PhabricatorFilesManagementEnginesWorkflow(),
35
+
new PhabricatorFilesManagementMigrateWorkflow(),
36
+
new PhutilHelpArgumentWorkflow(),
37
+
);
38
+
39
+
$args->parseWorkflows($workflows);
+6
src/__phutil_library_map__.php
+6
src/__phutil_library_map__.php
···
755
755
'PhabricatorFileUploadController' => 'applications/files/controller/PhabricatorFileUploadController.php',
756
756
'PhabricatorFileUploadException' => 'applications/files/exception/PhabricatorFileUploadException.php',
757
757
'PhabricatorFileUploadView' => 'applications/files/view/PhabricatorFileUploadView.php',
758
+
'PhabricatorFilesManagementEnginesWorkflow' => 'applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php',
759
+
'PhabricatorFilesManagementMigrateWorkflow' => 'applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php',
760
+
'PhabricatorFilesManagementWorkflow' => 'applications/files/management/PhabricatorFilesManagementWorkflow.php',
758
761
'PhabricatorFlag' => 'applications/flag/storage/PhabricatorFlag.php',
759
762
'PhabricatorFlagColor' => 'applications/flag/constants/PhabricatorFlagColor.php',
760
763
'PhabricatorFlagConstants' => 'applications/flag/constants/PhabricatorFlagConstants.php',
···
1949
1952
'PhabricatorFileUploadController' => 'PhabricatorFileController',
1950
1953
'PhabricatorFileUploadException' => 'Exception',
1951
1954
'PhabricatorFileUploadView' => 'AphrontView',
1955
+
'PhabricatorFilesManagementEnginesWorkflow' => 'PhabricatorFilesManagementWorkflow',
1956
+
'PhabricatorFilesManagementMigrateWorkflow' => 'PhabricatorFilesManagementWorkflow',
1957
+
'PhabricatorFilesManagementWorkflow' => 'PhutilArgumentWorkflow',
1952
1958
'PhabricatorFlag' => 'PhabricatorFlagDAO',
1953
1959
'PhabricatorFlagColor' => 'PhabricatorFlagConstants',
1954
1960
'PhabricatorFlagController' => 'PhabricatorController',
+46
src/applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php
+46
src/applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php
···
1
+
<?php
2
+
3
+
/*
4
+
* Copyright 2012 Facebook, Inc.
5
+
*
6
+
* Licensed under the Apache License, Version 2.0 (the "License");
7
+
* you may not use this file except in compliance with the License.
8
+
* You may obtain a copy of the License at
9
+
*
10
+
* http://www.apache.org/licenses/LICENSE-2.0
11
+
*
12
+
* Unless required by applicable law or agreed to in writing, software
13
+
* distributed under the License is distributed on an "AS IS" BASIS,
14
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+
* See the License for the specific language governing permissions and
16
+
* limitations under the License.
17
+
*/
18
+
19
+
final class PhabricatorFilesManagementEnginesWorkflow
20
+
extends PhabricatorFilesManagementWorkflow {
21
+
22
+
public function didConstruct() {
23
+
$this
24
+
->setName('engines')
25
+
->setSynopsis('List available storage engines.')
26
+
->setArguments(array());
27
+
}
28
+
29
+
public function execute(PhutilArgumentParser $args) {
30
+
$console = PhutilConsole::getConsole();
31
+
32
+
$engines = PhabricatorFile::buildAllEngines();
33
+
if (!$engines) {
34
+
throw new Exception("No storage engines are available.");
35
+
}
36
+
37
+
foreach ($engines as $engine) {
38
+
$console->writeOut(
39
+
"%s\n",
40
+
$engine->getEngineIdentifier());
41
+
}
42
+
43
+
return 0;
44
+
}
45
+
46
+
}
+153
src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php
+153
src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php
···
1
+
<?php
2
+
3
+
/*
4
+
* Copyright 2012 Facebook, Inc.
5
+
*
6
+
* Licensed under the Apache License, Version 2.0 (the "License");
7
+
* you may not use this file except in compliance with the License.
8
+
* You may obtain a copy of the License at
9
+
*
10
+
* http://www.apache.org/licenses/LICENSE-2.0
11
+
*
12
+
* Unless required by applicable law or agreed to in writing, software
13
+
* distributed under the License is distributed on an "AS IS" BASIS,
14
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+
* See the License for the specific language governing permissions and
16
+
* limitations under the License.
17
+
*/
18
+
19
+
final class PhabricatorFilesManagementMigrateWorkflow
20
+
extends PhabricatorFilesManagementWorkflow {
21
+
22
+
public function didConstruct() {
23
+
$this
24
+
->setName('migrate')
25
+
->setSynopsis('Migrate files between storage engines.')
26
+
->setArguments(
27
+
array(
28
+
array(
29
+
'name' => 'engine',
30
+
'param' => 'storage_engine',
31
+
'help' => 'Migrate to the named storage engine.',
32
+
),
33
+
array(
34
+
'name' => 'dry-run',
35
+
'help' => 'Show what would be migrated.',
36
+
),
37
+
array(
38
+
'name' => 'all',
39
+
'help' => 'Migrate all files.',
40
+
),
41
+
array(
42
+
'name' => 'names',
43
+
'wildcard' => true,
44
+
),
45
+
));
46
+
}
47
+
48
+
public function execute(PhutilArgumentParser $args) {
49
+
$console = PhutilConsole::getConsole();
50
+
51
+
$engine_id = $args->getArg('engine');
52
+
if (!$engine_id) {
53
+
throw new PhutilArgumentUsageException(
54
+
"Specify an engine to migrate to with `--engine`. ".
55
+
"Use `files engines` to get a list of engines.");
56
+
}
57
+
58
+
$engine = PhabricatorFile::buildEngine($engine_id);
59
+
60
+
if ($args->getArg('all')) {
61
+
if ($args->getArg('names')) {
62
+
throw new PhutilArgumentUsageException(
63
+
"Specify either a list of files or `--all`, but not both.");
64
+
}
65
+
$iterator = new LiskMigrationIterator(new PhabricatorFile());
66
+
} else if ($args->getArg('names')) {
67
+
$iterator = array();
68
+
69
+
foreach ($args->getArg('names') as $name) {
70
+
$name = trim($name);
71
+
72
+
$id = preg_replace('/^F/i', '', $name);
73
+
if (ctype_digit($id)) {
74
+
$file = id(new PhabricatorFile())->loadOneWhere(
75
+
'id = %d',
76
+
$id);
77
+
if (!$file) {
78
+
throw new PhutilArgumentUsageException(
79
+
"No file exists with id '{$name}'.");
80
+
}
81
+
} else {
82
+
$file = id(new PhabricatorFile())->loadOneWhere(
83
+
'phid = %d',
84
+
$name);
85
+
if (!$file) {
86
+
throw new PhutilArgumentUsageException(
87
+
"No file exists with PHID '{$name}'.");
88
+
}
89
+
}
90
+
$iterator[] = $file;
91
+
}
92
+
} else {
93
+
throw new PhutilArgumentUsageException(
94
+
"Either specify a list of files to migrate, or use `--all` ".
95
+
"to migrate all files.");
96
+
}
97
+
98
+
$is_dry_run = $args->getArg('dry-run');
99
+
100
+
$failed = array();
101
+
102
+
foreach ($iterator as $file) {
103
+
$fid = 'F'.$file->getID();
104
+
105
+
if ($file->getStorageEngine() == $engine_id) {
106
+
$console->writeOut(
107
+
"%s: Already stored on '%s'\n",
108
+
$fid,
109
+
$engine_id);
110
+
continue;
111
+
}
112
+
113
+
if ($is_dry_run) {
114
+
$console->writeOut(
115
+
"%s: Would migrate from '%s' to '%s' (dry run)\n",
116
+
$fid,
117
+
$file->getStorageEngine(),
118
+
$engine_id);
119
+
continue;
120
+
}
121
+
122
+
$console->writeOut(
123
+
"%s: Migrating from '%s' to '%s'...",
124
+
$fid,
125
+
$file->getStorageEngine(),
126
+
$engine_id);
127
+
128
+
try {
129
+
$file->migrateToEngine($engine);
130
+
$console->writeOut("done.\n");
131
+
} catch (Exception $ex) {
132
+
$console->writeOut("failed!\n");
133
+
$console->writeErr("%s\n", (string)$ex);
134
+
$failed[] = $file;
135
+
}
136
+
}
137
+
138
+
if ($failed) {
139
+
$console->writeOut("**Failures!**\n");
140
+
$ids = array();
141
+
foreach ($failed as $file) {
142
+
$ids[] = 'F'.$file->getID();
143
+
}
144
+
$console->writeOut("%s\n", implode(', ', $ids));
145
+
146
+
return 1;
147
+
} else {
148
+
$console->writeOut("**Success!**\n");
149
+
return 0;
150
+
}
151
+
}
152
+
153
+
}
+26
src/applications/files/management/PhabricatorFilesManagementWorkflow.php
+26
src/applications/files/management/PhabricatorFilesManagementWorkflow.php
···
1
+
<?php
2
+
3
+
/*
4
+
* Copyright 2012 Facebook, Inc.
5
+
*
6
+
* Licensed under the Apache License, Version 2.0 (the "License");
7
+
* you may not use this file except in compliance with the License.
8
+
* You may obtain a copy of the License at
9
+
*
10
+
* http://www.apache.org/licenses/LICENSE-2.0
11
+
*
12
+
* Unless required by applicable law or agreed to in writing, software
13
+
* distributed under the License is distributed on an "AS IS" BASIS,
14
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+
* See the License for the specific language governing permissions and
16
+
* limitations under the License.
17
+
*/
18
+
19
+
abstract class PhabricatorFilesManagementWorkflow
20
+
extends PhutilArgumentWorkflow {
21
+
22
+
public function isExecutable() {
23
+
return true;
24
+
}
25
+
26
+
}
+85
-27
src/applications/files/storage/PhabricatorFile.php
+85
-27
src/applications/files/storage/PhabricatorFile.php
···
149
149
throw new Exception("No valid storage engines are available!");
150
150
}
151
151
152
+
$file = new PhabricatorFile();
153
+
152
154
$data_handle = null;
153
155
$engine_identifier = null;
154
156
$exceptions = array();
155
157
foreach ($engines as $engine) {
156
158
$engine_class = get_class($engine);
157
159
try {
158
-
// Perform the actual write.
159
-
$data_handle = $engine->writeFile($data, $params);
160
-
if (!$data_handle || strlen($data_handle) > 255) {
161
-
// This indicates an improperly implemented storage engine.
162
-
throw new PhabricatorFileStorageConfigurationException(
163
-
"Storage engine '{$engine_class}' executed writeFile() but did ".
164
-
"not return a valid handle ('{$data_handle}') to the data: it ".
165
-
"must be nonempty and no longer than 255 characters.");
166
-
}
167
-
168
-
$engine_identifier = $engine->getEngineIdentifier();
169
-
if (!$engine_identifier || strlen($engine_identifier) > 32) {
170
-
throw new PhabricatorFileStorageConfigurationException(
171
-
"Storage engine '{$engine_class}' returned an improper engine ".
172
-
"identifier '{$engine_identifier}': it must be nonempty ".
173
-
"and no longer than 32 characters.");
174
-
}
160
+
list($engine_identifier, $data_handle) = $file->writeToEngine(
161
+
$engine,
162
+
$data,
163
+
$params);
175
164
176
165
// We stored the file somewhere so stop trying to write it to other
177
166
// places.
178
167
break;
179
-
180
168
} catch (PhabricatorFileStorageConfigurationException $ex) {
181
169
// If an engine is outright misconfigured (or misimplemented), raise
182
170
// that immediately since it probably needs attention.
183
171
throw $ex;
184
-
185
172
} catch (Exception $ex) {
173
+
phlog($ex);
174
+
186
175
// If an engine doesn't work, keep trying all the other valid engines
187
176
// in case something else works.
188
-
phlog($ex);
189
-
190
177
$exceptions[$engine_class] = $ex;
191
178
}
192
179
}
···
204
191
// (always the case with newFromFileDownload()), store a ''
205
192
$authorPHID = idx($params, 'authorPHID');
206
193
207
-
$file = new PhabricatorFile();
208
194
$file->setName($file_name);
209
195
$file->setByteSize(strlen($data));
210
196
$file->setAuthorPHID($authorPHID);
···
229
215
230
216
return $file;
231
217
}
218
+
219
+
public function migrateToEngine(PhabricatorFileStorageEngine $engine) {
220
+
if (!$this->getID() || !$this->getStorageHandle()) {
221
+
throw new Exception(
222
+
"You can not migrate a file which hasn't yet been saved.");
223
+
}
224
+
225
+
$data = $this->loadFileData();
226
+
$params = array(
227
+
'name' => $this->getName(),
228
+
);
229
+
230
+
list($new_identifier, $new_handle) = $this->writeToEngine(
231
+
$engine,
232
+
$data,
233
+
$params);
234
+
235
+
$old_engine = $this->instantiateStorageEngine();
236
+
$old_handle = $this->getStorageHandle();
237
+
238
+
$this->setStorageEngine($new_identifier);
239
+
$this->setStorageHandle($new_handle);
240
+
$this->save();
241
+
242
+
$old_engine->deleteFile($old_handle);
243
+
244
+
return $this;
245
+
}
246
+
247
+
private function writeToEngine(
248
+
PhabricatorFileStorageEngine $engine,
249
+
$data,
250
+
array $params) {
251
+
252
+
$engine_class = get_class($engine);
253
+
254
+
$data_handle = $engine->writeFile($data, $params);
255
+
256
+
if (!$data_handle || strlen($data_handle) > 255) {
257
+
// This indicates an improperly implemented storage engine.
258
+
throw new PhabricatorFileStorageConfigurationException(
259
+
"Storage engine '{$engine_class}' executed writeFile() but did ".
260
+
"not return a valid handle ('{$data_handle}') to the data: it ".
261
+
"must be nonempty and no longer than 255 characters.");
262
+
}
263
+
264
+
$engine_identifier = $engine->getEngineIdentifier();
265
+
if (!$engine_identifier || strlen($engine_identifier) > 32) {
266
+
throw new PhabricatorFileStorageConfigurationException(
267
+
"Storage engine '{$engine_class}' returned an improper engine ".
268
+
"identifier '{$engine_identifier}': it must be nonempty ".
269
+
"and no longer than 32 characters.");
270
+
}
271
+
272
+
return array($engine_identifier, $data_handle);
273
+
}
274
+
232
275
233
276
public static function newFromFileDownload($uri, $name) {
234
277
$uri = new PhutilURI($uri);
···
402
445
}
403
446
404
447
protected function instantiateStorageEngine() {
448
+
return self::buildEngine($this->getStorageEngine());
449
+
}
450
+
451
+
public static function buildEngine($engine_identifier) {
452
+
$engines = self::buildAllEngines();
453
+
foreach ($engines as $engine) {
454
+
if ($engine->getEngineIdentifier() == $engine_identifier) {
455
+
return $engine;
456
+
}
457
+
}
458
+
459
+
throw new Exception(
460
+
"Storage engine '{$engine_identifier}' could not be located!");
461
+
}
462
+
463
+
public static function buildAllEngines() {
405
464
$engines = id(new PhutilSymbolLoader())
406
465
->setType('class')
466
+
->setConcreteOnly(true)
407
467
->setAncestorClass('PhabricatorFileStorageEngine')
408
468
->selectAndLoadSymbols();
409
469
470
+
$results = array();
410
471
foreach ($engines as $engine_class) {
411
-
$engine = newv($engine_class['name'], array());
412
-
if ($engine->getEngineIdentifier() == $this->getStorageEngine()) {
413
-
return $engine;
414
-
}
472
+
$results[] = newv($engine_class['name'], array());
415
473
}
416
474
417
-
throw new Exception("File's storage engine could be located!");
475
+
return $results;
418
476
}
419
477
420
478
public function getViewableMimeType() {
+16
src/docs/configuration/configuring_file_storage.diviner
+16
src/docs/configuration/configuring_file_storage.diviner
···
80
80
You can test that things are correctly configured by going to the Files
81
81
application (##/file/##) and uploading files.
82
82
83
+
= Migrating Files Beteeen Engines =
84
+
85
+
If you want to move files between storage engines, you can use the `bin/files`
86
+
script to perform migrations. For example, suppose you previously used MySQL but
87
+
recently set up S3 and want to migrate all your files there. First, migrate one
88
+
file to make sure things work:
89
+
90
+
phabricator/ $ ./bin/files migrate --engine amazon-s3 F12345
91
+
92
+
If that works properly, you can then migrate everything:
93
+
94
+
phabricator/ $ ./bin/files migrate --engine amazon-s3 --all
95
+
96
+
You can use `--dry-run` to show which migrations would be performed without
97
+
taking any action. Run `bin/files help` for more options and information.
98
+
83
99
= Next Steps =
84
100
85
101
Continue by: