@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add "bin/file migrate" options to support import of a local-disk backup for Phacility instances
Summary:
Ref T13306. Currently, there's no easy way to import a third-party local-disk file dump into a Phacility instance.
Add some more options to `bin/files migrate` to support this. In particular, this enables:
```
$ ./bin/files --from-engine local-disk --engine amazon-s3 --local-disk-source path/to/backup
```
...to import these files into S3 directly.
These are general-purpose options and theoretically useful in other use cases, although realistically those cases are probably very rare.
Test Plan: Used `bin/files` with the new options to move files in and out of local disk storage in an arbitrary backup directory. Got clean exports/imports.
Reviewers: amckinley
Maniphest Tasks: T13306
Differential Revision: https://secure.phabricator.com/D20571
+83
-11
+34
src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php
+34
src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php
···
46
46
'name' => 'names',
47
47
'wildcard' => true,
48
48
),
49
+
array(
50
+
'name' => 'from-engine',
51
+
'param' => 'engine',
52
+
'help' => pht('Migrate files from the named storage engine.'),
53
+
),
54
+
array(
55
+
'name' => 'local-disk-source',
56
+
'param' => 'path',
57
+
'help' => pht(
58
+
'When migrating from a local disk source, use the specified '.
59
+
'path as the root directory.'),
60
+
),
49
61
));
50
62
}
51
63
52
64
public function execute(PhutilArgumentParser $args) {
65
+
66
+
// See T13306. This flag allows you to import files from a backup of
67
+
// local disk storage into some other engine. When the caller provides
68
+
// the flag, we override the local disk engine configuration and treat
69
+
// it as though it is configured to use the specified location.
70
+
71
+
$local_disk_source = $args->getArg('local-disk-source');
72
+
if (strlen($local_disk_source)) {
73
+
$path = Filesystem::resolvePath($local_disk_source);
74
+
try {
75
+
Filesystem::assertIsDirectory($path);
76
+
} catch (FilesystemException $ex) {
77
+
throw new PhutilArgumentUsageException(
78
+
pht(
79
+
'The "--local-disk-source" argument must point to a valid, '.
80
+
'readable directory on local disk.'));
81
+
}
82
+
83
+
$env = PhabricatorEnv::beginScopedEnv();
84
+
$env->overrideEnvConfig('storage.local-disk.path', $path);
85
+
}
86
+
53
87
$target_key = $args->getArg('engine');
54
88
if (!$target_key) {
55
89
throw new PhutilArgumentUsageException(
+36
-11
src/applications/files/management/PhabricatorFilesManagementWorkflow.php
+36
-11
src/applications/files/management/PhabricatorFilesManagementWorkflow.php
···
4
4
extends PhabricatorManagementWorkflow {
5
5
6
6
protected function buildIterator(PhutilArgumentParser $args) {
7
+
$viewer = $this->getViewer();
7
8
$names = $args->getArg('names');
8
9
9
-
if ($args->getArg('all')) {
10
-
if ($names) {
11
-
throw new PhutilArgumentUsageException(
12
-
pht(
13
-
'Specify either a list of files or `%s`, but not both.',
14
-
'--all'));
15
-
}
16
-
return new LiskMigrationIterator(new PhabricatorFile());
10
+
$is_all = $args->getArg('all');
11
+
$from_engine = $args->getArg('from-engine');
12
+
13
+
$any_constraint = ($from_engine || $names);
14
+
15
+
if (!$is_all && !$any_constraint) {
16
+
throw new PhutilArgumentUsageException(
17
+
pht(
18
+
'Use "--all" to migrate all files, or choose files to migrate '.
19
+
'with "--names" or "--from-engine".'));
20
+
}
21
+
22
+
if ($is_all && $any_constraint) {
23
+
throw new PhutilArgumentUsageException(
24
+
pht(
25
+
'You can not migrate all files with "--all" and also migrate only '.
26
+
'a subset of files with "--from-engine" or "--names".'));
17
27
}
18
28
29
+
// If we're migrating specific named files, convert the names into IDs
30
+
// first.
31
+
$ids = null;
19
32
if ($names) {
20
-
return $this->loadFilesWithNames($names);
33
+
$files = $this->loadFilesWithNames($names);
34
+
$ids = mpull($files, 'getID');
35
+
}
36
+
37
+
$query = id(new PhabricatorFileQuery())
38
+
->setViewer($viewer);
39
+
40
+
if ($ids) {
41
+
$query->withIDs($ids);
42
+
}
43
+
44
+
if ($from_engine) {
45
+
$query->withStorageEngines(array($from_engine));
21
46
}
22
47
23
-
return null;
48
+
return new PhabricatorQueryIterator($query);
24
49
}
25
50
26
51
protected function loadFilesWithNames(array $names) {
···
36
61
if (empty($files[$name])) {
37
62
throw new PhutilArgumentUsageException(
38
63
pht(
39
-
"No file '%s' exists!",
64
+
'No file "%s" exists.',
40
65
$name));
41
66
}
42
67
}
+13
src/applications/files/query/PhabricatorFileQuery.php
+13
src/applications/files/query/PhabricatorFileQuery.php
···
19
19
private $needTransforms;
20
20
private $builtinKeys;
21
21
private $isBuiltin;
22
+
private $storageEngines;
22
23
23
24
public function withIDs(array $ids) {
24
25
$this->ids = $ids;
···
135
136
return $this->withNgramsConstraint(
136
137
id(new PhabricatorFileNameNgrams()),
137
138
$ngrams);
139
+
}
140
+
141
+
public function withStorageEngines(array $engines) {
142
+
$this->storageEngines = $engines;
143
+
return $this;
138
144
}
139
145
140
146
public function showOnlyExplicitUploads($explicit_uploads) {
···
467
473
$conn,
468
474
'builtinKey IS NULL');
469
475
}
476
+
}
477
+
478
+
if ($this->storageEngines !== null) {
479
+
$where[] = qsprintf(
480
+
$conn,
481
+
'storageEngine IN (%Ls)',
482
+
$this->storageEngines);
470
483
}
471
484
472
485
return $where;