@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
1<?php
2
3/**
4 * @phutil-external-symbol class PhpParser\NodeTraverser
5 * @phutil-external-symbol class PhorgePHPParserExtractor
6 */
7final class PhabricatorInternationalizationManagementExtractWorkflow
8 extends PhabricatorInternationalizationManagementWorkflow {
9
10 const CACHE_VERSION = 1;
11
12 protected function didConstruct() {
13 $this
14 ->setName('extract')
15 ->setExamples(
16 '**extract** [__options__] __library__')
17 ->setSynopsis(pht('Extract translatable strings.'))
18 ->setArguments(
19 array(
20 array(
21 'name' => 'paths',
22 'wildcard' => true,
23 ),
24 array(
25 'name' => 'clean',
26 'help' => pht('Drop caches before extracting strings. Slow!'),
27 ),
28 ));
29 }
30
31 public function execute(PhutilArgumentParser $args) {
32 $console = PhutilConsole::getConsole();
33
34 $paths = $args->getArg('paths');
35 if (!$paths) {
36 $paths = array(getcwd());
37 }
38
39 $targets = array();
40 foreach ($paths as $path) {
41 $root = Filesystem::resolvePath($path);
42
43 if (!Filesystem::pathExists($root) || !is_dir($root)) {
44 throw new PhutilArgumentUsageException(
45 pht(
46 'Path "%s" does not exist, or is not a directory.',
47 $path));
48 }
49
50 $libraries = id(new FileFinder($path))
51 ->withPath('*/__phutil_library_init__.php')
52 ->find();
53 if (!$libraries) {
54 throw new PhutilArgumentUsageException(
55 pht(
56 'Path "%s" contains no libraries.',
57 $path));
58 }
59
60 foreach ($libraries as $library) {
61 $targets[] = Filesystem::resolvePath(dirname($path.'/'.$library)).'/';
62 }
63 }
64
65 $targets = array_unique($targets);
66
67 foreach ($targets as $library) {
68 echo tsprintf(
69 "**<bg:blue> %s </bg>** %s\n",
70 pht('EXTRACT'),
71 pht(
72 'Extracting "%s"...',
73 phutil_get_library_name_for_root($library) ??
74 Filesystem::readablePath($library)));
75
76 $this->extractLibrary($library);
77 }
78
79 return 0;
80 }
81
82 private function extractLibrary($root) {
83 $files = $this->loadLibraryFiles($root);
84 $cache = $this->readCache($root);
85
86 $modified = $this->getModifiedFiles($files, $cache);
87 $cache['files'] = $files;
88
89 if ($modified) {
90 echo tsprintf(
91 "**<bg:blue> %s </bg>** %s\n",
92 pht('MODIFIED'),
93 pht(
94 'Found %s modified file(s) (of %s total).',
95 phutil_count($modified),
96 phutil_count($files)));
97
98 $old_strings = idx($cache, 'strings');
99 $old_strings = array_select_keys($old_strings, $files);
100 $new_strings = $this->extractFiles($root, $modified);
101 $all_strings = $new_strings + $old_strings;
102 $cache['strings'] = $all_strings;
103
104 $this->writeStrings($root, $all_strings);
105 } else {
106 echo tsprintf(
107 "**<bg:blue> %s </bg>** %s\n",
108 pht('NOT MODIFIED'),
109 pht('Strings for this library are already up to date.'));
110 }
111
112 $cache = id(new PhutilJSON())->encodeFormatted($cache);
113 $this->writeCache($root, 'i18n_files.json', $cache);
114 }
115
116 private function getModifiedFiles(array $files, array $cache) {
117 $known = idx($cache, 'files', array());
118 $known = array_fuse($known);
119
120 $modified = array();
121 foreach ($files as $file => $hash) {
122
123 if (isset($known[$hash])) {
124 continue;
125 }
126 $modified[$file] = $hash;
127 }
128
129 return $modified;
130 }
131
132 private function extractFiles($root_path, array $files) {
133 $hashes = array();
134
135 $bar = id(new PhutilConsoleProgressBar())
136 ->setTotal(count($files));
137
138 $messages = array();
139 $results = array();
140
141 $parser = PhutilPHPParserLibrary::getParser();
142 // Load this class now (once PHP-Parser is built so its base class exists)
143 // this is not in the autoloader to avoid errors from tests that try
144 // to load every class without installing PHP-Parser
145 $root = dirname(phutil_get_library_root('phorge'));
146 require_once $root.'/support/php-parser/PhorgePHPParserExtractor.php';
147 foreach ($files as $file => $hash) {
148 $bar->update(1);
149 $full_path = $root_path.DIRECTORY_SEPARATOR.$file;
150 $hashes[$full_path] = $hash;
151 $file = Filesystem::readablePath($full_path, $root_path);
152 try {
153 $data = Filesystem::readFile($full_path);
154 $tree = $parser->parse($data);
155 $visitor = new PhorgePHPParserExtractor($file);
156 id(new PhpParser\NodeTraverser($visitor))->traverse($tree);
157 } catch (Exception $ex) {
158 $messages[] = pht(
159 'Failed to parse file "%s": %s',
160 $full_path,
161 $ex->getMessage());
162 continue;
163 }
164 $results[$hash] = $visitor->getResults();
165 $messages += $visitor->getWarnings();
166 }
167
168 $bar->done();
169
170 foreach ($messages as $message) {
171 echo tsprintf(
172 "**<bg:yellow> %s </bg>** %s\n",
173 pht('WARNING'),
174 $message);
175 }
176
177 return $results;
178 }
179
180 private function writeStrings($root, array $strings) {
181 $map = array();
182 foreach ($strings as $hash => $string_list) {
183 foreach ($string_list as $string_info) {
184 $string = $string_info['string'];
185
186 $map[$string]['uses'][] = array(
187 'file' => $string_info['file'],
188 'line' => $string_info['line'],
189 );
190
191 $stypes = $string_info['types'];
192 if (!isset($map[$string]['types'])) {
193 $map[$string]['types'] = $stypes;
194 } else if ($map[$string]['types'] !== $stypes) {
195 foreach ($map[$string]['types'] as $i => $t) {
196 if ($t !== $stypes[$i]) {
197 // This type is not consistent, set it to null since we don't know
198 $map[$string]['types'][$i] = null;
199 }
200 }
201 }
202 }
203 }
204
205 ksort($map);
206
207 $json = id(new PhutilJSON())->encodeFormatted($map);
208 $this->writeCache($root, 'i18n_strings.json', $json);
209 }
210
211 private function loadLibraryFiles($root) {
212 $files = $this->loadDirectoryFiles($root);
213 $extra_dirs = array('support', 'scripts');
214 foreach ($extra_dirs as $extra) {
215 $extra = '..'.DIRECTORY_SEPARATOR.$extra.DIRECTORY_SEPARATOR;
216 if (Filesystem::pathExists(Filesystem::resolvePath($root.$extra))) {
217 $files = array_merge($files, $this->loadDirectoryFiles($root, $extra));
218 }
219 }
220 return $files;
221 }
222
223 private function loadDirectoryFiles($root, $suffix = '') {
224 $files = id(new FileFinder($root.$suffix))
225 ->withType('f')
226 ->withSuffix('php')
227 ->excludePath('*/.*')
228 ->excludePath('*/__tests__/*')
229 ->setGenerateChecksums(true)
230 ->find();
231
232 $map = array();
233 foreach ($files as $file => $hash) {
234 $file = Filesystem::readablePath($file, $root);
235 $file = ltrim($file, '/');
236
237 if (dirname($file) == '.') {
238 continue;
239 }
240
241 if (dirname($file) == 'extensions') {
242 continue;
243 }
244
245 $map[$suffix.$file] = md5($hash.$file);
246 }
247
248 return $map;
249 }
250
251 private function readCache($root) {
252 $path = $this->getCachePath($root, 'i18n_files.json');
253
254 $default = array(
255 'version' => self::CACHE_VERSION,
256 'files' => array(),
257 'strings' => array(),
258 );
259
260 if ($this->getArgv()->getArg('clean')) {
261 return $default;
262 }
263
264 if (!Filesystem::pathExists($path)) {
265 return $default;
266 }
267
268 try {
269 $data = Filesystem::readFile($path);
270 } catch (Exception $ex) {
271 return $default;
272 }
273
274 try {
275 $cache = phutil_json_decode($data);
276 } catch (PhutilJSONParserException $e) {
277 return $default;
278 }
279
280 $version = idx($cache, 'version');
281 if ($version !== self::CACHE_VERSION) {
282 return $default;
283 }
284
285 return $cache;
286 }
287
288 private function writeCache($root, $file, $data) {
289 $path = $this->getCachePath($root, $file);
290
291 $cache_dir = dirname($path);
292 if (!Filesystem::pathExists($cache_dir)) {
293 Filesystem::createDirectory($cache_dir, 0755, true);
294 }
295
296 Filesystem::writeFile($path, $data);
297 }
298
299 private function getCachePath($root, $to_file) {
300 return $root.'/.cache/'.$to_file;
301 }
302
303}