@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
1<?php
2
3final class PhabricatorFilesManagementMigrateWorkflow
4 extends PhabricatorFilesManagementWorkflow {
5
6 protected function didConstruct() {
7 $arguments = $this->newIteratorArguments();
8
9 $arguments[] = array(
10 'name' => 'engine',
11 'param' => 'storage-engine',
12 'help' => pht('Migrate to the named storage engine.'),
13 );
14
15 $arguments[] = array(
16 'name' => 'dry-run',
17 'help' => pht('Show what would be migrated.'),
18 );
19
20 $arguments[] = array(
21 'name' => 'min-size',
22 'param' => 'bytes',
23 'help' => pht(
24 'Do not migrate data for files which are smaller than a given '.
25 'filesize.'),
26 );
27
28 $arguments[] = array(
29 'name' => 'max-size',
30 'param' => 'bytes',
31 'help' => pht(
32 'Do not migrate data for files which are larger than a given '.
33 'filesize.'),
34 );
35
36 $arguments[] = array(
37 'name' => 'copy',
38 'help' => pht(
39 'Copy file data instead of moving it: after migrating, do not '.
40 'remove the old data even if it is no longer referenced.'),
41 );
42
43 $arguments[] = array(
44 'name' => 'local-disk-source',
45 'param' => 'path',
46 'help' => pht(
47 'When migrating from a local disk source, use the specified '.
48 'path as the root directory.'),
49 );
50
51 $this
52 ->setName('migrate')
53 ->setSynopsis(pht('Migrate files between storage engines.'))
54 ->setArguments($arguments);
55 }
56
57 public function execute(PhutilArgumentParser $args) {
58
59 // See T13306. This flag allows you to import files from a backup of
60 // local disk storage into some other engine. When the caller provides
61 // the flag, we override the local disk engine configuration and treat
62 // it as though it is configured to use the specified location.
63
64 $local_disk_source = $args->getArg('local-disk-source');
65 if (phutil_nonempty_string($local_disk_source)) {
66 $path = Filesystem::resolvePath($local_disk_source);
67 try {
68 Filesystem::assertIsDirectory($path);
69 } catch (FilesystemException $ex) {
70 throw new PhutilArgumentUsageException(
71 pht(
72 'The "--local-disk-source" argument must point to a valid, '.
73 'readable directory on local disk.'));
74 }
75
76 $env = PhabricatorEnv::beginScopedEnv();
77 $env->overrideEnvConfig('storage.local-disk.path', $path);
78 }
79
80 $target_key = $args->getArg('engine');
81 if (!$target_key) {
82 throw new PhutilArgumentUsageException(
83 pht(
84 'Specify an engine to migrate to with `%s`. '.
85 'Use `%s` to get a list of engines.',
86 '--engine',
87 'files engines'));
88 }
89
90 $target_engine = PhabricatorFile::buildEngine($target_key);
91
92 $iterator = $this->buildIterator($args);
93 $is_dry_run = $args->getArg('dry-run');
94
95 $min_size = (int)$args->getArg('min-size');
96 $max_size = (int)$args->getArg('max-size');
97
98 $is_copy = $args->getArg('copy');
99
100 $failed = array();
101 $engines = PhabricatorFileStorageEngine::loadAllEngines();
102 $total_bytes = 0;
103 $total_files = 0;
104 foreach ($iterator as $file) {
105 $monogram = $file->getMonogram();
106
107 // See T7148. When we export data for an instance, we copy all the data
108 // for Files from S3 into the database dump so that the database dump is
109 // a complete, standalone archive of all the data. In the general case,
110 // installs may have a similar process using "--copy" to create a more
111 // complete backup.
112
113 // When doing this, we may run into temporary files which have been
114 // deleted between the time we took the original dump and the current
115 // timestamp. These files can't be copied since the data no longer
116 // exists: the daemons on the live install already deleted it.
117
118 // Simply avoid this whole mess by declining to migrate expired temporary
119 // files. They're as good as dead anyway.
120
121 $ttl = $file->getTTL();
122 if ($ttl) {
123 if ($ttl < PhabricatorTime::getNow()) {
124 echo tsprintf(
125 "%s\n",
126 pht(
127 '%s: Skipping expired temporary file.',
128 $monogram));
129 continue;
130 }
131 }
132
133 $engine_key = $file->getStorageEngine();
134 $engine = idx($engines, $engine_key);
135
136 if (!$engine) {
137 echo tsprintf(
138 "%s\n",
139 pht(
140 '%s: Uses unknown storage engine "%s".',
141 $monogram,
142 $engine_key));
143 $failed[] = $file;
144 continue;
145 }
146
147 if ($engine->isChunkEngine()) {
148 echo tsprintf(
149 "%s\n",
150 pht(
151 '%s: Stored as chunks, no data to migrate directly.',
152 $monogram));
153 continue;
154 }
155
156 if ($engine_key === $target_key) {
157 echo tsprintf(
158 "%s\n",
159 pht(
160 '%s: Already stored in engine "%s".',
161 $monogram,
162 $target_key));
163 continue;
164 }
165
166 $byte_size = $file->getByteSize();
167
168 if ($min_size && ($byte_size < $min_size)) {
169 echo tsprintf(
170 "%s\n",
171 pht(
172 '%s: File size (%s) is smaller than minimum size (%s).',
173 $monogram,
174 phutil_format_bytes($byte_size),
175 phutil_format_bytes($min_size)));
176 continue;
177 }
178
179 if ($max_size && ($byte_size > $max_size)) {
180 echo tsprintf(
181 "%s\n",
182 pht(
183 '%s: File size (%s) is larger than maximum size (%s).',
184 $monogram,
185 phutil_format_bytes($byte_size),
186 phutil_format_bytes($max_size)));
187 continue;
188 }
189
190 if ($is_dry_run) {
191 echo tsprintf(
192 "%s\n",
193 pht(
194 '%s: (%s) Would migrate from "%s" to "%s" (dry run)...',
195 $monogram,
196 phutil_format_bytes($byte_size),
197 $engine_key,
198 $target_key));
199 } else {
200 echo tsprintf(
201 "%s\n",
202 pht(
203 '%s: (%s) Migrating from "%s" to "%s"...',
204 $monogram,
205 phutil_format_bytes($byte_size),
206 $engine_key,
207 $target_key));
208 }
209
210 try {
211 if ($is_dry_run) {
212 // Do nothing, this is a dry run.
213 } else {
214 $file->migrateToEngine($target_engine, $is_copy);
215 }
216
217 $total_files += 1;
218 $total_bytes += $byte_size;
219
220 echo tsprintf(
221 "%s\n",
222 pht('Done.'));
223
224 } catch (Exception $ex) {
225 echo tsprintf(
226 "%s\n",
227 pht('Failed! %s', (string)$ex));
228 $failed[] = $file;
229
230 throw $ex;
231 }
232 }
233
234 echo tsprintf(
235 "%s\n",
236 pht(
237 'Total Migrated Files: %s',
238 new PhutilNumber($total_files)));
239
240 echo tsprintf(
241 "%s\n",
242 pht(
243 'Total Migrated Bytes: %s',
244 phutil_format_bytes($total_bytes)));
245
246 if ($is_dry_run) {
247 echo tsprintf(
248 "%s\n",
249 pht(
250 'This was a dry run, so no real migrations were performed.'));
251 }
252
253 if ($failed) {
254 $monograms = mpull($failed, 'getMonogram');
255
256 echo tsprintf(
257 "%s\n",
258 pht('Failures: %s.', implode(', ', $monograms)));
259
260 return 1;
261 }
262
263 return 0;
264 }
265
266}