@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add a `bin/i18n validate` workflow to check for errors in translation files
Summary:
Part of T16378. Depends on D26559 (for the `getAllTranslations` method)
Someday I may turn this into a unit test that ensures there are no validation errors. But not yet - the patches to fix all of the validation errors are still in review or yet to be written.
Test Plan: Run the script via `./bin/i18n validate`, see various errors. Apply various in-review patches (D26548, D26558, D26546, D26574) and see fewer errors with each patch applied. With all of those patches applied (and the downstream translations extension not installed since it doesn't follow these conventions yet) see no validation errors at all.
Reviewers: O1 Blessed Committers, aklapper
Reviewed By: O1 Blessed Committers, aklapper
Subscribers: aklapper, tobiaswiese, valerio.bozzolan, Matthew, Cigaryno
Differential Revision: https://we.phorge.it/D26572
+232
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
5413
5413
'PhorgeCodeWarningSetupCheck' => 'applications/config/check/PhorgeCodeWarningSetupCheck.php',
5414
5414
'PhorgeFlagFlaggedObjectCustomField' => 'applications/flag/customfield/PhorgeFlagFlaggedObjectCustomField.php',
5415
5415
'PhorgeFlagFlaggedObjectFieldStorage' => 'applications/flag/customfield/PhorgeFlagFlaggedObjectFieldStorage.php',
5416
+
'PhorgeInternationalizationManagementValidateWorkflow' => 'infrastructure/internationalization/management/PhorgeInternationalizationManagementValidateWorkflow.php',
5417
+
'PhorgeInternationalizationValidator' => 'infrastructure/internationalization/management/PhorgeInternationalizationValidator.php',
5416
5418
'PhorgePHPASTParseTree' => 'applications/phpast/storage/PhorgePHPASTParseTree.php',
5417
5419
'PhorgePHPASTViewFrameController' => 'applications/phpast/controller/PhorgePHPASTViewFrameController.php',
5418
5420
'PhorgePHPASTViewFramesetController' => 'applications/phpast/controller/PhorgePHPASTViewFramesetController.php',
···
12285
12287
'PhorgeCodeWarningSetupCheck' => 'PhabricatorSetupCheck',
12286
12288
'PhorgeFlagFlaggedObjectCustomField' => 'PhabricatorCustomField',
12287
12289
'PhorgeFlagFlaggedObjectFieldStorage' => 'Phobject',
12290
+
'PhorgeInternationalizationManagementValidateWorkflow' => 'PhabricatorInternationalizationManagementWorkflow',
12291
+
'PhorgeInternationalizationValidator' => 'Phobject',
12288
12292
'PhorgePHPASTParseTree' => 'PhabricatorXHPASTDAO',
12289
12293
'PhorgePHPASTViewFrameController' => 'PhabricatorXHPASTViewController',
12290
12294
'PhorgePHPASTViewFramesetController' => 'PhabricatorXHPASTViewController',
+38
src/infrastructure/internationalization/management/PhorgeInternationalizationManagementValidateWorkflow.php
+38
src/infrastructure/internationalization/management/PhorgeInternationalizationManagementValidateWorkflow.php
···
1
+
<?php
2
+
3
+
final class PhorgeInternationalizationManagementValidateWorkflow
4
+
extends PhabricatorInternationalizationManagementWorkflow {
5
+
6
+
protected function didConstruct() {
7
+
$this
8
+
->setName('validate')
9
+
->setExamples(
10
+
'**validate** [__options__]')
11
+
->setSynopsis(pht(
12
+
'Validate that all locales and translations are properly constructed.'))
13
+
->setArguments(
14
+
array(
15
+
array(
16
+
'name' => 'extract',
17
+
'help' => pht(
18
+
'Do an implicit string extraction before validating. '.
19
+
'If this is not set, it will validate based on the last '.
20
+
'extracted strings.'),
21
+
),
22
+
));
23
+
}
24
+
public function execute(PhutilArgumentParser $args) {
25
+
$validator = new PhorgeInternationalizationValidator();
26
+
$do_extract = $args->getArg('extract');
27
+
$json = $validator->loadExtractions($do_extract);
28
+
$errors = $validator->validateLibraries($json);
29
+
if (!count($errors)) {
30
+
echo pht('No validation errors found!').PHP_EOL;
31
+
} else {
32
+
echo pht('The following validation errors were found:').PHP_EOL;
33
+
foreach ($errors as $error) {
34
+
echo $error.PHP_EOL;
35
+
}
36
+
}
37
+
}
38
+
}
+166
src/infrastructure/internationalization/management/PhorgeInternationalizationValidator.php
+166
src/infrastructure/internationalization/management/PhorgeInternationalizationValidator.php
···
1
+
<?php
2
+
3
+
final class PhorgeInternationalizationValidator extends Phobject {
4
+
private function validateTranslation($locale, $proto, $transl, $types, $n) {
5
+
$errors = array();
6
+
if ($n >= count($types)) {
7
+
if (is_array($transl)) {
8
+
$errors[] = pht(
9
+
'The locale `%s` defines a translation for the key `%s`, '.
10
+
'which has at least %s level(s) of arrays, '.
11
+
'however the source message has only %s parameter(s).',
12
+
$locale,
13
+
$proto,
14
+
new PhutilNumber($n + 1),
15
+
phutil_count($types));
16
+
}
17
+
$data = [];
18
+
foreach ($types as $type) {
19
+
$data[] = $type === 'number' ? 3: 'abc';
20
+
}
21
+
try {
22
+
$parsed = vsprintf($transl, $data);
23
+
} catch (ValueError $ex) {
24
+
// In PHP 8 vsprintf throws a ValueError for bad data;
25
+
// in PHP7 it returns false
26
+
$parsed = false;
27
+
}
28
+
if ($parsed === false) {
29
+
$errors[] = pht(
30
+
'The locale `%s` defines a translation for the key `%s` which '.
31
+
'failed to interpolate properly. Probably it defines too many '.
32
+
'parameters.',
33
+
$locale,
34
+
$proto);
35
+
}
36
+
return $errors;
37
+
}
38
+
if (!is_array($transl)) {
39
+
$transl = array($transl);
40
+
}
41
+
$type = $types[$n];
42
+
if ($type === null && count($transl) != 1) {
43
+
$errors[] = pht(
44
+
'The locale `%s` defines a translation for the key `%s`, which varies '.
45
+
'on the plurality or gender of parameter %d, however that parameter '.
46
+
'is not a number or person.',
47
+
$locale,
48
+
$proto,
49
+
$n + 1);
50
+
}
51
+
foreach ($transl as $subarray) {
52
+
$errors = array_merge(
53
+
$errors,
54
+
$this->validateTranslation($locale, $proto, $subarray, $types, $n + 1));
55
+
}
56
+
return $errors;
57
+
}
58
+
public function validateLibraries($loaded_json) {
59
+
$errors = [];
60
+
$all_translations = PhutilTranslation::getAllTranslations();
61
+
$locales = PhutilLocale::loadAllLocales();
62
+
$keyed_translations = [];
63
+
$override_key = 'translation.override';
64
+
try {
65
+
$trans_override = PhabricatorEnv::getEnvConfig($override_key);
66
+
$all_translations[$override_key] = $trans_override;
67
+
} catch (Throwable $ex) {
68
+
// If Phorge config is hosed then just don't check translation.override
69
+
}
70
+
foreach ($all_translations as $locale_code => $translations) {
71
+
if (!isset($locales[$locale_code]) && $locale_code != $override_key) {
72
+
$errors[] = pht(
73
+
'Translations are defined for the locale `%s`, '.
74
+
'which is not recognized.',
75
+
$locale_code);
76
+
}
77
+
foreach ($translations as $proto => $transl) {
78
+
if (!isset($keyed_translations[$proto])) {
79
+
$keyed_translations[$proto] = array();
80
+
}
81
+
$keyed_translations[$proto][$locale_code] = $transl;
82
+
// Check for unused translations
83
+
if (!isset($loaded_json[$proto])) {
84
+
$errors[] = pht(
85
+
'The locale `%s` defines a translation for the string "%s", '.
86
+
'however that string does not appear to be referenced '.
87
+
'by the codebase.',
88
+
$locale_code,
89
+
$proto);
90
+
}
91
+
}
92
+
}
93
+
foreach ($loaded_json as $string => $spec) {
94
+
// Check for wrong branches by parameter type
95
+
$translations = idx($keyed_translations, $string, array());
96
+
foreach ($translations as $locale => $translation) {
97
+
$errors = array_merge($errors, $this->validateTranslation(
98
+
$locale,
99
+
$string,
100
+
$translation,
101
+
$spec['types'],
102
+
0));
103
+
}
104
+
// Check for missing branches in US english
105
+
if (str_contains($string, '(s)')) {
106
+
if (!isset($keyed_translations[$string]['en_US'])) {
107
+
foreach ($spec['types'] as $type) {
108
+
if ($type === 'number') {
109
+
$errors[] = pht(
110
+
'The string "%s" contains the placeholder "(s)" and '.
111
+
'a numeric parameter on which to vary the (s) by, '.
112
+
'however the builtin US English translation does not do so.',
113
+
$string);
114
+
}
115
+
}
116
+
}
117
+
}
118
+
}
119
+
return $errors;
120
+
}
121
+
public function loadExtractions($run_extractor) {
122
+
$libraries = PhutilBootloader::getInstance()->getAllLibraries();
123
+
$phorge_root = phutil_get_library_root('phorge');
124
+
$i18n_bin = $phorge_root.'/../bin/i18n';
125
+
$all_json = [];
126
+
foreach ($libraries as $lib) {
127
+
$root = phutil_get_library_root($lib);
128
+
$json = Filesystem::resolvePath($root.'/.cache/i18n_strings.json');
129
+
if ($run_extractor) {
130
+
$err = phutil_passthru(
131
+
'%R extract %s',
132
+
$i18n_bin,
133
+
$root);
134
+
if ($err) {
135
+
throw new Exception(pht(
136
+
'Failed to run i18n extractor: %s',
137
+
$err));
138
+
}
139
+
} else if (!Filesystem::pathExists($json)) {
140
+
throw new Exception(pht(
141
+
'Strings have not yet been extracted for library %s. '.
142
+
'Run `%s` for that library first to extract them or '.
143
+
're-run with `%s` to automatically extract missing strngs.',
144
+
$lib,
145
+
'bin/i18n extract',
146
+
'--extract'));
147
+
}
148
+
$all_json += phutil_json_decode(Filesystem::readFile($json));
149
+
}
150
+
// Add extra date elements from PhutilTranslator::translateDate
151
+
// which aren't in a static pht() call
152
+
$date = array('types' => array());
153
+
$all_json['Jan'] = $date;
154
+
$all_json['Feb'] = $date;
155
+
$all_json['Mar'] = $date;
156
+
$all_json['Apr'] = $date;
157
+
$all_json['Jun'] = $date;
158
+
$all_json['Jul'] = $date;
159
+
$all_json['Aug'] = $date;
160
+
$all_json['Sep'] = $date;
161
+
$all_json['Oct'] = $date;
162
+
$all_json['Nov'] = $date;
163
+
$all_json['Dec'] = $date;
164
+
return $all_json;
165
+
}
166
+
}
+24
src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php
+24
src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php
···
2364
2364
'%s Read [%s bytes]',
2365
2365
),
2366
2366
),
2367
+
'The locale `%s` defines a translation for the key `%s`, which has '.
2368
+
'at least %s level(s) of arrays, however the source message has only '.
2369
+
'%s parameter(s).' => array(
2370
+
array(
2371
+
array(
2372
+
array(
2373
+
'The locale `%s` defines a translation for the key `%s`, which '.
2374
+
'has at least %s level of arrays, however the source message '.
2375
+
'has only %s parameter.',
2376
+
'The locale `%s` defines a translation for the key `%s`, which '.
2377
+
'has at least %s level of arrays, however the source message '.
2378
+
'has only %s parameters.',
2379
+
),
2380
+
array(
2381
+
'The locale `%s` defines a translation for the key `%s`, which '.
2382
+
'has at least %s levels of arrays, however the source message '.
2383
+
'has only %s parameter.',
2384
+
'The locale `%s` defines a translation for the key `%s`, which '.
2385
+
'has at least %s levels of arrays, however the source message '.
2386
+
'has only %s parameters.',
2387
+
),
2388
+
),
2389
+
),
2390
+
),
2367
2391
);
2368
2392
}
2369
2393
}