loading up the forgejo repo on tangled to test page performance
chore: teach lint-locale about locale_next (#6800)
- Ref: forgejo/forgejo#6203 & forgejo/forgejo#5703
- Moved code around to be reusable, otherwise an straightforward implementation.
- Added unit test.
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6800
Reviewed-by: 0ko <0ko@noreply.codeberg.org>
Co-authored-by: Gusted <postmaster@gusted.xyz>
Co-committed-by: Gusted <postmaster@gusted.xyz>
+99
-10
+70
-10
build/lint-locale.go
+70
-10
build/lint-locale.go
···
5
5
package main
6
6
7
7
import (
8
+
"encoding/json" //nolint:depguard
8
9
"fmt"
9
10
"html"
10
11
"io/fs"
···
26
27
27
28
// Matches href="", href="#", href="%s", href="#%s", href="%[1]s" and href="#%[1]s".
28
29
placeHolderRegex = regexp.MustCompile(`href="#?(%s|%\[\d\]s)?"`)
30
+
31
+
dmp = diffmatchpatch.New()
29
32
)
30
33
31
34
func initBlueMondayPolicy() {
···
79
82
return value
80
83
}
81
84
85
+
func checkValue(trKey, value string) []string {
86
+
keyValue := preprocessTranslationValue(value)
87
+
88
+
if html.UnescapeString(policy.Sanitize(keyValue)) == keyValue {
89
+
return nil
90
+
}
91
+
92
+
// Create a nice diff of the difference.
93
+
diffs := dmp.DiffMain(keyValue, html.UnescapeString(policy.Sanitize(keyValue)), false)
94
+
diffs = dmp.DiffCleanupSemantic(diffs)
95
+
diffs = dmp.DiffCleanupEfficiency(diffs)
96
+
97
+
return []string{trKey + ": " + dmp.DiffPrettyText(diffs)}
98
+
}
99
+
82
100
func checkLocaleContent(localeContent []byte) []string {
83
101
// Same configuration as Forgejo uses.
84
102
cfg := ini.Empty(ini.LoadOptions{
···
90
108
panic(err)
91
109
}
92
110
93
-
dmp := diffmatchpatch.New()
94
111
errors := []string{}
95
-
96
112
for _, section := range cfg.Sections() {
97
113
for _, key := range section.Keys() {
98
114
var trKey string
···
102
118
trKey = section.Name() + "." + key.Name()
103
119
}
104
120
105
-
keyValue := preprocessTranslationValue(key.Value())
121
+
errors = append(errors, checkValue(trKey, key.Value())...)
122
+
}
123
+
}
124
+
return errors
125
+
}
106
126
107
-
if html.UnescapeString(policy.Sanitize(keyValue)) != keyValue {
108
-
// Create a nice diff of the difference.
109
-
diffs := dmp.DiffMain(keyValue, html.UnescapeString(policy.Sanitize(keyValue)), false)
110
-
diffs = dmp.DiffCleanupSemantic(diffs)
111
-
diffs = dmp.DiffCleanupEfficiency(diffs)
127
+
func checkLocaleNextContent(data map[string]any, trKey ...string) []string {
128
+
errors := []string{}
129
+
for key, value := range data {
130
+
currentKey := key
131
+
if len(trKey) == 1 {
132
+
currentKey = trKey[0] + "." + key
133
+
}
112
134
113
-
errors = append(errors, trKey+": "+dmp.DiffPrettyText(diffs))
114
-
}
135
+
switch value := value.(type) {
136
+
case string:
137
+
errors = append(errors, checkValue(currentKey, value)...)
138
+
case map[string]any:
139
+
errors = append(errors, checkLocaleNextContent(value, currentKey)...)
140
+
default:
141
+
panic(fmt.Sprintf("Unexpected type during linting locale next: %s - %T", currentKey, value))
115
142
}
116
143
}
117
144
return errors
···
127
154
panic(err)
128
155
}
129
156
157
+
// Safety check that we are not reading the wrong directory.
130
158
if !slices.ContainsFunc(localeFiles, func(e fs.DirEntry) bool { return strings.HasSuffix(e.Name(), ".ini") }) {
131
159
fmt.Println("No locale files found")
132
160
os.Exit(1)
···
144
172
}
145
173
146
174
if err := checkLocaleContent(localeContent); len(err) > 0 {
175
+
fmt.Println(localeFile.Name())
176
+
fmt.Println(strings.Join(err, "\n"))
177
+
fmt.Println()
178
+
exitCode = 1
179
+
}
180
+
}
181
+
182
+
// Check the locale next.
183
+
localeDir = filepath.Join("options", "locale_next")
184
+
localeFiles, err = os.ReadDir(localeDir)
185
+
if err != nil {
186
+
panic(err)
187
+
}
188
+
189
+
// Safety check that we are not reading the wrong directory.
190
+
if !slices.ContainsFunc(localeFiles, func(e fs.DirEntry) bool { return strings.HasSuffix(e.Name(), ".json") }) {
191
+
fmt.Println("No locale_next files found")
192
+
os.Exit(1)
193
+
}
194
+
195
+
for _, localeFile := range localeFiles {
196
+
localeContent, err := os.ReadFile(filepath.Join(localeDir, localeFile.Name()))
197
+
if err != nil {
198
+
panic(err)
199
+
}
200
+
201
+
var localeData map[string]any
202
+
if err := json.Unmarshal(localeContent, &localeData); err != nil {
203
+
panic(err)
204
+
}
205
+
206
+
if err := checkLocaleNextContent(localeData); len(err) > 0 {
147
207
fmt.Println(localeFile.Name())
148
208
fmt.Println(strings.Join(err, "\n"))
149
209
fmt.Println()
+29
build/lint-locale_test.go
+29
build/lint-locale_test.go
···
63
63
assert.EqualValues(t, []string{"key: و\x1b[31m \x1b[0m\x1b[32m\u00a0\x1b[0m"}, checkLocaleContent([]byte(`key = و `)))
64
64
})
65
65
}
66
+
67
+
func TestNextLocalizationPolicy(t *testing.T) {
68
+
initBlueMondayPolicy()
69
+
initRemoveTags()
70
+
71
+
t.Run("Nested locales", func(t *testing.T) {
72
+
assert.Empty(t, checkLocaleNextContent(map[string]any{
73
+
"settings": map[string]any{
74
+
"hidden_comment_types_description": `Comment types checked here will not be shown inside issue pages. Checking "Label" for example removes all "<user> added/removed <label>" comments.`,
75
+
},
76
+
}))
77
+
78
+
assert.EqualValues(t, []string{"settings.hidden_comment_types_description: \"\x1b[31m<not-an-allowed-key>\x1b[0m REPLACED-TAG\""}, checkLocaleNextContent(map[string]any{
79
+
"settings": map[string]any{
80
+
"hidden_comment_types_description": `"<not-an-allowed-key> <label>"`,
81
+
},
82
+
}))
83
+
})
84
+
85
+
t.Run("Flat locales", func(t *testing.T) {
86
+
assert.Empty(t, checkLocaleNextContent(map[string]any{
87
+
"settings.hidden_comment_types_description": `Comment types checked here will not be shown inside issue pages. Checking "Label" for example removes all "<user> added/removed <label>" comments.`,
88
+
}))
89
+
90
+
assert.EqualValues(t, []string{"settings.hidden_comment_types_description: \"\x1b[31m<not-an-allowed-key>\x1b[0m REPLACED-TAG\""}, checkLocaleNextContent(map[string]any{
91
+
"settings.hidden_comment_types_description": `"<not-an-allowed-key> <label>"`,
92
+
}))
93
+
})
94
+
}