loading up the forgejo repo on tangled to test page performance
feat(build): teach lint-locale-usage about trPluralString (#7425)
This requires using the more complicated parsing from localestore.go
In order to avoid future code drift and code duplication,
localestore.go was refactored to call IterateMessagesContent instead of
essentially duplicating the code of RecursivelyAddTranslationsFromJSON
with small adjustments.
locale/utils.go was moved to translation/localeiter/utils.go
in order to avoid spreading translation-related routines among completely
different places.
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/7425
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
Co-authored-by: Ellen Emilia Anna Zscheile <fogti+devel@ytrizja.de>
Co-committed-by: Ellen Emilia Anna Zscheile <fogti+devel@ytrizja.de>
authored by
Ellen Emilia Anna Zscheile
Ellen Emilia Anna Zscheile
and committed by
Gusted
15a2338f
bd9366e7
+136
-129
+3
.deadcode-out
+3
.deadcode-out
+10
-7
build/lint-locale-usage/lint-locale-usage.go
+10
-7
build/lint-locale-usage/lint-locale-usage.go
···
18
18
tmplParser "text/template/parse"
19
19
20
20
"forgejo.org/modules/container"
21
-
"forgejo.org/modules/locale"
22
21
fjTemplates "forgejo.org/modules/templates"
22
+
"forgejo.org/modules/translation/localeiter"
23
23
"forgejo.org/modules/util"
24
24
)
25
25
···
300
300
}
301
301
302
302
msgids := make(container.Set[string])
303
-
onMsgid := func(trKey, trValue string) error {
304
-
msgids[trKey] = struct{}{}
305
-
return nil
306
-
}
307
303
308
304
localeFile := filepath.Join(filepath.Join("options", "locale"), "locale_en-US.ini")
309
305
localeContent, err := os.ReadFile(localeFile)
···
312
308
os.Exit(2)
313
309
}
314
310
315
-
if err = locale.IterateMessagesContent(localeContent, onMsgid); err != nil {
311
+
if err = localeiter.IterateMessagesContent(localeContent, func(trKey, trValue string) error {
312
+
msgids[trKey] = struct{}{}
313
+
return nil
314
+
}); err != nil {
316
315
fmt.Printf("%s:\tERROR: %s\n", localeFile, err.Error())
317
316
os.Exit(2)
318
317
}
···
324
323
os.Exit(2)
325
324
}
326
325
327
-
if err := locale.IterateMessagesNextContent(localeContent, onMsgid); err != nil {
326
+
if err := localeiter.IterateMessagesNextContent(localeContent, func(trKey, pluralForm, trValue string) error {
327
+
// ignore plural form
328
+
msgids[trKey] = struct{}{}
329
+
return nil
330
+
}); err != nil {
328
331
fmt.Printf("%s:\tERROR: %s\n", localeFile, err.Error())
329
332
os.Exit(2)
330
333
}
+8
-4
build/lint-locale/lint-locale.go
+8
-4
build/lint-locale/lint-locale.go
···
14
14
"slices"
15
15
"strings"
16
16
17
-
"forgejo.org/modules/locale"
17
+
"forgejo.org/modules/translation/localeiter"
18
18
19
19
"github.com/microcosm-cc/bluemonday"
20
20
"github.com/sergi/go-diff/diffmatchpatch"
···
100
100
func checkLocaleContent(localeContent []byte) []string {
101
101
errors := []string{}
102
102
103
-
if err := locale.IterateMessagesContent(localeContent, func(trKey, trValue string) error {
103
+
if err := localeiter.IterateMessagesContent(localeContent, func(trKey, trValue string) error {
104
104
errors = append(errors, checkValue(trKey, trValue)...)
105
105
return nil
106
106
}); err != nil {
···
113
113
func checkLocaleNextContent(localeContent []byte) []string {
114
114
errors := []string{}
115
115
116
-
if err := locale.IterateMessagesNextContent(localeContent, func(trKey, trValue string) error {
117
-
errors = append(errors, checkValue(trKey, trValue)...)
116
+
if err := localeiter.IterateMessagesNextContent(localeContent, func(trKey, pluralForm, trValue string) error {
117
+
fullKey := trKey
118
+
if pluralForm != "" {
119
+
fullKey = trKey + "." + pluralForm
120
+
}
121
+
errors = append(errors, checkValue(fullKey, trValue)...)
118
122
return nil
119
123
}); err != nil {
120
124
panic(err)
+12
build/lint-locale/lint-locale_test.go
+12
build/lint-locale/lint-locale_test.go
···
91
91
"settings.hidden_comment_types_description": "\"<not-an-allowed-key> <label>\""
92
92
}`)))
93
93
})
94
+
95
+
t.Run("Plural form", func(t *testing.T) {
96
+
assert.Equal(t, []string{"repo.pulls.title_desc: key = \x1b[31m<a href=\"https://example.com\">\x1b[0m"}, checkLocaleNextContent([]byte(`{"repo.pulls.title_desc": {
97
+
"few": "key = <a href=\"%s\">",
98
+
"other": "key = <a href=\"https://example.com\">"
99
+
}}`)))
100
+
101
+
assert.Equal(t, []string{"repo.pulls.title_desc.few: key = \x1b[31m<a href=\"https://example.com\">\x1b[0m"}, checkLocaleNextContent([]byte(`{"repo.pulls.title_desc": {
102
+
"few": "key = <a href=\"https://example.com\">",
103
+
"other": "key = <a href=\"%s\">"
104
+
}}`)))
105
+
})
94
106
}
-74
modules/locale/utils.go
-74
modules/locale/utils.go
···
1
-
// Copyright 2023 The Gitea Authors. All rights reserved.
2
-
// Copyright 2025 The Forgejo Authors. All rights reserved.
3
-
// SPDX-License-Identifier: MIT
4
-
5
-
// extracted from `/build/lint-locale.go`, `/build/lint-locale-usage.go`
6
-
7
-
package locale
8
-
9
-
import (
10
-
"encoding/json" //nolint:depguard
11
-
"fmt"
12
-
13
-
"gopkg.in/ini.v1" //nolint:depguard
14
-
)
15
-
16
-
func IterateMessagesContent(localeContent []byte, onMsgid func(string, string) error) error {
17
-
// Same configuration as Forgejo uses.
18
-
cfg := ini.Empty(ini.LoadOptions{
19
-
IgnoreContinuation: true,
20
-
})
21
-
cfg.NameMapper = ini.SnackCase
22
-
23
-
if err := cfg.Append(localeContent); err != nil {
24
-
return err
25
-
}
26
-
27
-
for _, section := range cfg.Sections() {
28
-
for _, key := range section.Keys() {
29
-
var trKey string
30
-
if section.Name() == "" || section.Name() == "DEFAULT" || section.Name() == "common" {
31
-
trKey = key.Name()
32
-
} else {
33
-
trKey = section.Name() + "." + key.Name()
34
-
}
35
-
if err := onMsgid(trKey, key.Value()); err != nil {
36
-
return err
37
-
}
38
-
}
39
-
}
40
-
41
-
return nil
42
-
}
43
-
44
-
func iterateMessagesNextInner(onMsgid func(string, string) error, data map[string]any, trKey ...string) error {
45
-
for key, value := range data {
46
-
currentKey := key
47
-
if len(trKey) == 1 {
48
-
currentKey = trKey[0] + "." + key
49
-
}
50
-
51
-
switch value := value.(type) {
52
-
case string:
53
-
if err := onMsgid(currentKey, value); err != nil {
54
-
return err
55
-
}
56
-
case map[string]any:
57
-
if err := iterateMessagesNextInner(onMsgid, value, currentKey); err != nil {
58
-
return err
59
-
}
60
-
default:
61
-
return fmt.Errorf("unexpected type: %s - %T", currentKey, value)
62
-
}
63
-
}
64
-
65
-
return nil
66
-
}
67
-
68
-
func IterateMessagesNextContent(localeContent []byte, onMsgid func(string, string) error) error {
69
-
var localeData map[string]any
70
-
if err := json.Unmarshal(localeContent, &localeData); err != nil {
71
-
return err
72
-
}
73
-
return iterateMessagesNextInner(onMsgid, localeData)
74
-
}
+9
-44
modules/translation/i18n/localestore.go
+9
-44
modules/translation/i18n/localestore.go
···
8
8
"html/template"
9
9
"slices"
10
10
11
-
"forgejo.org/modules/json"
12
11
"forgejo.org/modules/log"
13
12
"forgejo.org/modules/setting"
13
+
"forgejo.org/modules/translation/localeiter"
14
14
"forgejo.org/modules/util"
15
15
)
16
16
···
94
94
return nil
95
95
}
96
96
97
-
func RecursivelyAddTranslationsFromJSON(locale *locale, object map[string]any, prefix string) error {
98
-
for key, value := range object {
99
-
var fullkey string
100
-
if prefix != "" {
101
-
fullkey = prefix + "." + key
102
-
} else {
103
-
fullkey = key
104
-
}
105
-
106
-
switch v := value.(type) {
107
-
case string:
108
-
// Check whether we are adding a plural form to the parent object, or a new nested JSON object.
109
-
110
-
switch key {
111
-
case "zero", "one", "two", "few", "many":
112
-
locale.newStyleMessages[prefix+PluralFormSeparator+key] = v
113
-
case "other":
114
-
locale.newStyleMessages[prefix] = v
115
-
default:
116
-
locale.newStyleMessages[fullkey] = v
117
-
}
118
-
119
-
case map[string]any:
120
-
err := RecursivelyAddTranslationsFromJSON(locale, v, fullkey)
121
-
if err != nil {
122
-
return err
123
-
}
124
-
125
-
case nil:
126
-
default:
127
-
return fmt.Errorf("Unrecognized JSON value '%s'", value)
128
-
}
129
-
}
130
-
131
-
return nil
132
-
}
133
-
134
97
func (store *localeStore) AddToLocaleFromJSON(langName string, source []byte) error {
135
98
locale, ok := store.localeMap[langName]
136
99
if !ok {
137
100
return ErrLocaleDoesNotExist
138
101
}
139
102
140
-
var result map[string]any
141
-
if err := json.Unmarshal(source, &result); err != nil {
142
-
return err
143
-
}
144
-
145
-
return RecursivelyAddTranslationsFromJSON(locale, result, "")
103
+
return localeiter.IterateMessagesNextContent(source, func(key, pluralForm, value string) error {
104
+
msgKey := key
105
+
if pluralForm != "" {
106
+
msgKey = key + PluralFormSeparator + pluralForm
107
+
}
108
+
locale.newStyleMessages[msgKey] = value
109
+
return nil
110
+
})
146
111
}
147
112
148
113
func (l *locale) LookupNewStyleMessage(trKey string) string {
+94
modules/translation/localeiter/utils.go
+94
modules/translation/localeiter/utils.go
···
1
+
// Copyright 2023 The Gitea Authors. All rights reserved.
2
+
// Copyright 2025 The Forgejo Authors. All rights reserved.
3
+
// SPDX-License-Identifier: MIT
4
+
5
+
// extracted from `/build/lint-locale.go`, `/build/lint-locale-usage.go`
6
+
7
+
package localeiter
8
+
9
+
import (
10
+
"encoding/json" //nolint:depguard
11
+
"fmt"
12
+
13
+
"gopkg.in/ini.v1" //nolint:depguard
14
+
)
15
+
16
+
func IterateMessagesContent(localeContent []byte, onMsgid func(string, string) error) error {
17
+
// Same configuration as Forgejo uses.
18
+
cfg := ini.Empty(ini.LoadOptions{
19
+
IgnoreContinuation: true,
20
+
})
21
+
cfg.NameMapper = ini.SnackCase
22
+
23
+
if err := cfg.Append(localeContent); err != nil {
24
+
return err
25
+
}
26
+
27
+
for _, section := range cfg.Sections() {
28
+
for _, key := range section.Keys() {
29
+
var trKey string
30
+
// see https://codeberg.org/forgejo/discussions/issues/104
31
+
// https://github.com/WeblateOrg/weblate/issues/10831
32
+
// for an explanation of why "common" is an alternative
33
+
if section.Name() == "" || section.Name() == "DEFAULT" || section.Name() == "common" {
34
+
trKey = key.Name()
35
+
} else {
36
+
trKey = section.Name() + "." + key.Name()
37
+
}
38
+
if err := onMsgid(trKey, key.Value()); err != nil {
39
+
return err
40
+
}
41
+
}
42
+
}
43
+
44
+
return nil
45
+
}
46
+
47
+
func iterateMessagesNextInner(onMsgid func(string, string, string) error, data map[string]any, trKey string) error {
48
+
for key, value := range data {
49
+
fullKey := key
50
+
if trKey != "" {
51
+
fullKey = trKey + "." + key
52
+
}
53
+
switch value := value.(type) {
54
+
case string:
55
+
// Check whether we are adding a plural form to the parent object, or a new nested JSON object.
56
+
realKey := trKey
57
+
pluralSuffix := ""
58
+
59
+
switch key {
60
+
case "zero", "one", "two", "few", "many":
61
+
pluralSuffix = key
62
+
case "other":
63
+
// do nothing
64
+
default:
65
+
realKey = fullKey
66
+
}
67
+
68
+
if err := onMsgid(realKey, pluralSuffix, value); err != nil {
69
+
return err
70
+
}
71
+
72
+
case map[string]any:
73
+
if err := iterateMessagesNextInner(onMsgid, value, fullKey); err != nil {
74
+
return err
75
+
}
76
+
77
+
case nil:
78
+
// do nothing
79
+
80
+
default:
81
+
return fmt.Errorf("Unexpected JSON type: %s - %T", fullKey, value)
82
+
}
83
+
}
84
+
85
+
return nil
86
+
}
87
+
88
+
func IterateMessagesNextContent(localeContent []byte, onMsgid func(string, string, string) error) error {
89
+
var localeData map[string]any
90
+
if err := json.Unmarshal(localeContent, &localeData); err != nil {
91
+
return err
92
+
}
93
+
return iterateMessagesNextInner(onMsgid, localeData, "")
94
+
}