@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Make i18n string extraction faster and more flexible
Summary:
Ref T5267. Two general changes:
- Make string extraction use a cache, so that it doesn't take several minutes every time you change something. Minor updates now only take a few seconds (like `arc liberate` and similar).
- Instead of dumping a sort-of-template file out, write out to a cache (`src/.cache/i18n_strings.json`). I'm planning to add more steps to read this cache and do interesting things with it (emit translatewiki strings, generate or update standalone translation files, etc).
Test Plan:
- Ran `bin/i18n extract`.
- Ran it again, saw it go a lot faster.
- Changed stuff, ran it, saw it only look at new stuff.
- Examined caches.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T5267
Differential Revision: https://secure.phabricator.com/D16227
+227
-44
+1
.gitignore
+1
.gitignore
+226
-44
src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php
+226
-44
src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php
···
3
3
final class PhabricatorInternationalizationManagementExtractWorkflow
4
4
extends PhabricatorInternationalizationManagementWorkflow {
5
5
6
+
const CACHE_VERSION = 1;
7
+
6
8
protected function didConstruct() {
7
9
$this
8
10
->setName('extract')
11
+
->setExamples(
12
+
'**extract** [__options__] __library__')
9
13
->setSynopsis(pht('Extract translatable strings.'))
10
14
->setArguments(
11
15
array(
···
13
17
'name' => 'paths',
14
18
'wildcard' => true,
15
19
),
20
+
array(
21
+
'name' => 'clean',
22
+
'help' => pht('Drop caches before extracting strings. Slow!'),
23
+
),
16
24
));
17
25
}
18
26
19
27
public function execute(PhutilArgumentParser $args) {
20
28
$console = PhutilConsole::getConsole();
29
+
21
30
$paths = $args->getArg('paths');
31
+
if (!$paths) {
32
+
$paths = array(getcwd());
33
+
}
22
34
23
-
$futures = array();
35
+
$targets = array();
24
36
foreach ($paths as $path) {
25
37
$root = Filesystem::resolvePath($path);
26
-
$path_files = id(new FileFinder($root))
27
-
->withType('f')
28
-
->withSuffix('php')
38
+
39
+
if (!Filesystem::pathExists($root) || !is_dir($root)) {
40
+
throw new PhutilArgumentUsageException(
41
+
pht(
42
+
'Path "%s" does not exist, or is not a directory.',
43
+
$path));
44
+
}
45
+
46
+
$libraries = id(new FileFinder($path))
47
+
->withPath('*/__phutil_library_init__.php')
29
48
->find();
49
+
if (!$libraries) {
50
+
throw new PhutilArgumentUsageException(
51
+
pht(
52
+
'Path "%s" contains no libphutil libraries.',
53
+
$path));
54
+
}
30
55
31
-
foreach ($path_files as $file) {
32
-
$full_path = $root.DIRECTORY_SEPARATOR.$file;
33
-
$data = Filesystem::readFile($full_path);
34
-
$futures[$full_path] = PhutilXHPASTBinary::getParserFuture($data);
56
+
foreach ($libraries as $library) {
57
+
$targets[] = Filesystem::resolvePath(dirname($library)).'/';
58
+
}
59
+
}
60
+
61
+
$targets = array_unique($targets);
62
+
63
+
foreach ($targets as $library) {
64
+
echo tsprintf(
65
+
"**<bg:blue> %s </bg>** %s\n",
66
+
pht('EXTRACT'),
67
+
pht(
68
+
'Extracting "%s"...',
69
+
Filesystem::readablePath($library)));
70
+
71
+
$this->extractLibrary($library);
72
+
}
73
+
74
+
return 0;
75
+
}
76
+
77
+
private function extractLibrary($root) {
78
+
$files = $this->loadLibraryFiles($root);
79
+
$cache = $this->readCache($root);
80
+
81
+
$modified = $this->getModifiedFiles($files, $cache);
82
+
$cache['files'] = $files;
83
+
84
+
if ($modified) {
85
+
echo tsprintf(
86
+
"**<bg:blue> %s </bg>** %s\n",
87
+
pht('MODIFIED'),
88
+
pht(
89
+
'Found %s modified file(s) (of %s total).',
90
+
phutil_count($modified),
91
+
phutil_count($files)));
92
+
93
+
$old_strings = idx($cache, 'strings');
94
+
$old_strings = array_select_keys($old_strings, $files);
95
+
$new_strings = $this->extractFiles($root, $modified);
96
+
$all_strings = $new_strings + $old_strings;
97
+
$cache['strings'] = $all_strings;
98
+
99
+
$this->writeStrings($root, $all_strings);
100
+
} else {
101
+
echo tsprintf(
102
+
"**<bg:blue> %s </bg>** %s\n",
103
+
pht('NOT MODIFIED'),
104
+
pht('Strings for this library are already up to date.'));
105
+
}
106
+
107
+
$cache = id(new PhutilJSON())->encodeFormatted($cache);
108
+
$this->writeCache($root, 'i18n_files.json', $cache);
109
+
}
110
+
111
+
private function getModifiedFiles(array $files, array $cache) {
112
+
$known = idx($cache, 'files', array());
113
+
$known = array_fuse($known);
114
+
115
+
$modified = array();
116
+
foreach ($files as $file => $hash) {
117
+
118
+
if (isset($known[$hash])) {
119
+
continue;
35
120
}
121
+
$modified[$file] = $hash;
36
122
}
37
123
38
-
$console->writeErr(
39
-
"%s\n",
40
-
pht('Found %s file(s)...', phutil_count($futures)));
124
+
return $modified;
125
+
}
126
+
127
+
private function extractFiles($root_path, array $files) {
128
+
$hashes = array();
129
+
130
+
$futures = array();
131
+
foreach ($files as $file => $hash) {
132
+
$full_path = $root_path.DIRECTORY_SEPARATOR.$file;
133
+
$data = Filesystem::readFile($full_path);
134
+
$futures[$full_path] = PhutilXHPASTBinary::getParserFuture($data);
41
135
42
-
$results = array();
136
+
$hashes[$full_path] = $hash;
137
+
}
43
138
44
139
$bar = id(new PhutilConsoleProgressBar())
45
140
->setTotal(count($futures));
46
141
47
142
$messages = array();
143
+
$results = array();
48
144
49
145
$futures = id(new FutureIterator($futures))
50
146
->limit(8);
51
147
foreach ($futures as $full_path => $future) {
52
148
$bar->update(1);
149
+
150
+
$hash = $hashes[$full_path];
53
151
54
152
try {
55
153
$tree = XHPASTTree::newFromDataAndResolvedExecFuture(
···
67
165
$calls = $root->selectDescendantsOfType('n_FUNCTION_CALL');
68
166
foreach ($calls as $call) {
69
167
$name = $call->getChildByIndex(0)->getConcreteString();
70
-
if ($name == 'pht') {
71
-
$params = $call->getChildByIndex(1, 'n_CALL_PARAMETER_LIST');
72
-
$string_node = $params->getChildByIndex(0);
73
-
$string_line = $string_node->getLineNumber();
74
-
try {
75
-
$string_value = $string_node->evalStatic();
168
+
if ($name != 'pht') {
169
+
continue;
170
+
}
76
171
77
-
$results[$string_value][] = array(
78
-
'file' => Filesystem::readablePath($full_path),
79
-
'line' => $string_line,
80
-
);
81
-
} catch (Exception $ex) {
82
-
$messages[] = pht(
83
-
'WARNING: Failed to evaluate pht() call on line %d in "%s": %s',
84
-
$call->getLineNumber(),
85
-
$full_path,
86
-
$ex->getMessage());
87
-
}
172
+
$params = $call->getChildByIndex(1, 'n_CALL_PARAMETER_LIST');
173
+
$string_node = $params->getChildByIndex(0);
174
+
$string_line = $string_node->getLineNumber();
175
+
try {
176
+
$string_value = $string_node->evalStatic();
177
+
178
+
$results[$hash][] = array(
179
+
'string' => $string_value,
180
+
'file' => Filesystem::readablePath($full_path, $root_path),
181
+
'line' => $string_line,
182
+
);
183
+
} catch (Exception $ex) {
184
+
$messages[] = pht(
185
+
'WARNING: Failed to evaluate pht() call on line %d in "%s": %s',
186
+
$call->getLineNumber(),
187
+
$full_path,
188
+
$ex->getMessage());
88
189
}
89
190
}
90
191
···
93
194
$bar->done();
94
195
95
196
foreach ($messages as $message) {
96
-
$console->writeErr("%s\n", $message);
197
+
echo tsprintf(
198
+
"**<bg:yellow> %s </bg>** %s\n",
199
+
pht('WARNING'),
200
+
$message);
97
201
}
98
202
99
-
ksort($results);
203
+
return $results;
204
+
}
100
205
101
-
$out = array();
102
-
$out[] = '<?php';
103
-
$out[] = '// @no'.'lint';
104
-
$out[] = 'return array(';
105
-
foreach ($results as $string => $locations) {
106
-
foreach ($locations as $location) {
107
-
$out[] = ' // '.$location['file'].':'.$location['line'];
206
+
private function writeStrings($root, array $strings) {
207
+
$map = array();
208
+
foreach ($strings as $hash => $string_list) {
209
+
foreach ($string_list as $string_info) {
210
+
$map[$string_info['string']]['uses'][] = array(
211
+
'file' => $string_info['file'],
212
+
'line' => $string_info['line'],
213
+
);
214
+
}
215
+
}
216
+
217
+
ksort($map);
218
+
219
+
$json = id(new PhutilJSON())->encodeFormatted($map);
220
+
$this->writeCache($root, 'i18n_strings.json', $json);
221
+
}
222
+
223
+
private function loadLibraryFiles($root) {
224
+
$files = id(new FileFinder($root))
225
+
->withType('f')
226
+
->withSuffix('php')
227
+
->excludePath('*/.*')
228
+
->setGenerateChecksums(true)
229
+
->find();
230
+
231
+
$map = array();
232
+
foreach ($files as $file => $hash) {
233
+
$file = Filesystem::readablePath($file, $root);
234
+
$file = ltrim($file, '/');
235
+
236
+
if (dirname($file) == '.') {
237
+
continue;
238
+
}
239
+
240
+
if (dirname($file) == 'extensions') {
241
+
continue;
108
242
}
109
-
$out[] = " '".addcslashes($string, "\0..\37\\'\177..\377")."' => null,";
110
-
$out[] = null;
243
+
244
+
$map[$file] = md5($hash.$file);
245
+
}
246
+
247
+
return $map;
248
+
}
249
+
250
+
private function readCache($root) {
251
+
$path = $this->getCachePath($root, 'i18n_files.json');
252
+
253
+
$default = array(
254
+
'version' => self::CACHE_VERSION,
255
+
'files' => array(),
256
+
'strings' => array(),
257
+
);
258
+
259
+
if ($this->getArgv()->getArg('clean')) {
260
+
return $default;
261
+
}
262
+
263
+
if (!Filesystem::pathExists($path)) {
264
+
return $default;
265
+
}
266
+
267
+
try {
268
+
$data = Filesystem::readFile($path);
269
+
} catch (Exception $ex) {
270
+
return $default;
271
+
}
272
+
273
+
try {
274
+
$cache = phutil_json_decode($data);
275
+
} catch (PhutilJSONParserException $e) {
276
+
return $default;
277
+
}
278
+
279
+
$version = idx($cache, 'version');
280
+
if ($version !== self::CACHE_VERSION) {
281
+
return $default;
282
+
}
283
+
284
+
return $cache;
285
+
}
286
+
287
+
private function writeCache($root, $file, $data) {
288
+
$path = $this->getCachePath($root, $file);
289
+
290
+
$cache_dir = dirname($path);
291
+
if (!Filesystem::pathExists($cache_dir)) {
292
+
Filesystem::createDirectory($cache_dir, 0755, true);
111
293
}
112
-
$out[] = ');';
113
-
$out[] = null;
114
294
115
-
echo implode("\n", $out);
295
+
Filesystem::writeFile($path, $data);
296
+
}
116
297
117
-
return 0;
298
+
private function getCachePath($root, $to_file) {
299
+
return $root.'/.cache/'.$to_file;
118
300
}
119
301
120
302
}