Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
scripts/kernel-doc.py: move regex methods to a separate file
In preparation for letting kerneldoc Sphinx extension to import
Python libraries, move regex ancillary classes to a separate
file.
Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
Signed-off-by: Jonathan Corbet <corbet@lwn.net>
Link: https://lore.kernel.org/r/64f96b6744435b51894bb4ab7612851d9d054190.1744106241.git.mchehab+huawei@kernel.org
authored by
Mauro Carvalho Chehab
and committed by
Jonathan Corbet
e31fd36d
01d3235d
+277
-218
+5
-218
scripts/kernel-doc.py
+5
-218
scripts/kernel-doc.py
···
110
110
111
111
from dateutil import tz
112
112
113
-
# Local cache for regular expressions
114
-
re_cache = {}
113
+
# Import Python modules
115
114
115
+
LIB_DIR = "lib/kdoc"
116
+
SRC_DIR = os.path.dirname(os.path.realpath(__file__))
116
117
117
-
class Re:
118
-
"""
119
-
Helper class to simplify regex declaration and usage,
118
+
sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR))
120
119
121
-
It calls re.compile for a given pattern. It also allows adding
122
-
regular expressions and define sub at class init time.
120
+
from kdoc_re import Re, NestedMatch
123
121
124
-
Regular expressions can be cached via an argument, helping to speedup
125
-
searches.
126
-
"""
127
-
128
-
def _add_regex(self, string, flags):
129
-
if string in re_cache:
130
-
self.regex = re_cache[string]
131
-
else:
132
-
self.regex = re.compile(string, flags=flags)
133
-
134
-
if self.cache:
135
-
re_cache[string] = self.regex
136
-
137
-
def __init__(self, string, cache=True, flags=0):
138
-
self.cache = cache
139
-
self.last_match = None
140
-
141
-
self._add_regex(string, flags)
142
-
143
-
def __str__(self):
144
-
return self.regex.pattern
145
-
146
-
def __add__(self, other):
147
-
return Re(str(self) + str(other), cache=self.cache or other.cache,
148
-
flags=self.regex.flags | other.regex.flags)
149
-
150
-
def match(self, string):
151
-
self.last_match = self.regex.match(string)
152
-
return self.last_match
153
-
154
-
def search(self, string):
155
-
self.last_match = self.regex.search(string)
156
-
return self.last_match
157
-
158
-
def findall(self, string):
159
-
return self.regex.findall(string)
160
-
161
-
def split(self, string):
162
-
return self.regex.split(string)
163
-
164
-
def sub(self, sub, string, count=0):
165
-
return self.regex.sub(sub, string, count=count)
166
-
167
-
def group(self, num):
168
-
return self.last_match.group(num)
169
-
170
-
class NestedMatch:
171
-
"""
172
-
Finding nested delimiters is hard with regular expressions. It is
173
-
even harder on Python with its normal re module, as there are several
174
-
advanced regular expressions that are missing.
175
-
176
-
This is the case of this pattern:
177
-
178
-
'\\bSTRUCT_GROUP(\\(((?:(?>[^)(]+)|(?1))*)\\))[^;]*;'
179
-
180
-
which is used to properly match open/close parenthesis of the
181
-
string search STRUCT_GROUP(),
182
-
183
-
Add a class that counts pairs of delimiters, using it to match and
184
-
replace nested expressions.
185
-
186
-
The original approach was suggested by:
187
-
https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex
188
-
189
-
Although I re-implemented it to make it more generic and match 3 types
190
-
of delimiters. The logic checks if delimiters are paired. If not, it
191
-
will ignore the search string.
192
-
"""
193
-
194
-
# TODO:
195
-
# Right now, regular expressions to match it are defined only up to
196
-
# the start delimiter, e.g.:
197
-
#
198
-
# \bSTRUCT_GROUP\(
199
-
#
200
-
# is similar to: STRUCT_GROUP\((.*)\)
201
-
# except that the content inside the match group is delimiter's aligned.
202
-
#
203
-
# The content inside parenthesis are converted into a single replace
204
-
# group (e.g. r`\1').
205
-
#
206
-
# It would be nice to change such definition to support multiple
207
-
# match groups, allowing a regex equivalent to.
208
-
#
209
-
# FOO\((.*), (.*), (.*)\)
210
-
#
211
-
# it is probably easier to define it not as a regular expression, but
212
-
# with some lexical definition like:
213
-
#
214
-
# FOO(arg1, arg2, arg3)
215
-
216
-
217
-
DELIMITER_PAIRS = {
218
-
'{': '}',
219
-
'(': ')',
220
-
'[': ']',
221
-
}
222
-
223
-
RE_DELIM = re.compile(r'[\{\}\[\]\(\)]')
224
-
225
-
def _search(self, regex, line):
226
-
"""
227
-
Finds paired blocks for a regex that ends with a delimiter.
228
-
229
-
The suggestion of using finditer to match pairs came from:
230
-
https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex
231
-
but I ended using a different implementation to align all three types
232
-
of delimiters and seek for an initial regular expression.
233
-
234
-
The algorithm seeks for open/close paired delimiters and place them
235
-
into a stack, yielding a start/stop position of each match when the
236
-
stack is zeroed.
237
-
238
-
The algorithm shoud work fine for properly paired lines, but will
239
-
silently ignore end delimiters that preceeds an start delimiter.
240
-
This should be OK for kernel-doc parser, as unaligned delimiters
241
-
would cause compilation errors. So, we don't need to rise exceptions
242
-
to cover such issues.
243
-
"""
244
-
245
-
stack = []
246
-
247
-
for match_re in regex.finditer(line):
248
-
start = match_re.start()
249
-
offset = match_re.end()
250
-
251
-
d = line[offset -1]
252
-
if d not in self.DELIMITER_PAIRS:
253
-
continue
254
-
255
-
end = self.DELIMITER_PAIRS[d]
256
-
stack.append(end)
257
-
258
-
for match in self.RE_DELIM.finditer(line[offset:]):
259
-
pos = match.start() + offset
260
-
261
-
d = line[pos]
262
-
263
-
if d in self.DELIMITER_PAIRS:
264
-
end = self.DELIMITER_PAIRS[d]
265
-
266
-
stack.append(end)
267
-
continue
268
-
269
-
# Does the end delimiter match what it is expected?
270
-
if stack and d == stack[-1]:
271
-
stack.pop()
272
-
273
-
if not stack:
274
-
yield start, offset, pos + 1
275
-
break
276
-
277
-
def search(self, regex, line):
278
-
"""
279
-
This is similar to re.search:
280
-
281
-
It matches a regex that it is followed by a delimiter,
282
-
returning occurrences only if all delimiters are paired.
283
-
"""
284
-
285
-
for t in self._search(regex, line):
286
-
287
-
yield line[t[0]:t[2]]
288
-
289
-
def sub(self, regex, sub, line, count=0):
290
-
"""
291
-
This is similar to re.sub:
292
-
293
-
It matches a regex that it is followed by a delimiter,
294
-
replacing occurrences only if all delimiters are paired.
295
-
296
-
if r'\1' is used, it works just like re: it places there the
297
-
matched paired data with the delimiter stripped.
298
-
299
-
If count is different than zero, it will replace at most count
300
-
items.
301
-
"""
302
-
out = ""
303
-
304
-
cur_pos = 0
305
-
n = 0
306
-
307
-
found = False
308
-
for start, end, pos in self._search(regex, line):
309
-
out += line[cur_pos:start]
310
-
311
-
# Value, ignoring start/end delimiters
312
-
value = line[end:pos - 1]
313
-
314
-
# replaces \1 at the sub string, if \1 is used there
315
-
new_sub = sub
316
-
new_sub = new_sub.replace(r'\1', value)
317
-
318
-
out += new_sub
319
-
320
-
# Drop end ';' if any
321
-
if line[pos] == ';':
322
-
pos += 1
323
-
324
-
cur_pos = pos
325
-
n += 1
326
-
327
-
if count and count >= n:
328
-
break
329
-
330
-
# Append the remaining string
331
-
l = len(line)
332
-
out += line[cur_pos:l]
333
-
334
-
return out
335
122
336
123
#
337
124
# Regular expressions used to parse kernel-doc markups at KernelDoc class.
+272
scripts/lib/kdoc/kdoc_re.py
+272
scripts/lib/kdoc/kdoc_re.py
···
1
+
#!/usr/bin/env python3
2
+
# SPDX-License-Identifier: GPL-2.0
3
+
# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>.
4
+
5
+
"""
6
+
Regular expression ancillary classes.
7
+
8
+
Those help caching regular expressions and do matching for kernel-doc.
9
+
"""
10
+
11
+
import re
12
+
13
+
# Local cache for regular expressions
14
+
re_cache = {}
15
+
16
+
17
+
class Re:
18
+
"""
19
+
Helper class to simplify regex declaration and usage,
20
+
21
+
It calls re.compile for a given pattern. It also allows adding
22
+
regular expressions and define sub at class init time.
23
+
24
+
Regular expressions can be cached via an argument, helping to speedup
25
+
searches.
26
+
"""
27
+
28
+
def _add_regex(self, string, flags):
29
+
"""
30
+
Adds a new regex or re-use it from the cache.
31
+
"""
32
+
33
+
if string in re_cache:
34
+
self.regex = re_cache[string]
35
+
else:
36
+
self.regex = re.compile(string, flags=flags)
37
+
38
+
if self.cache:
39
+
re_cache[string] = self.regex
40
+
41
+
def __init__(self, string, cache=True, flags=0):
42
+
"""
43
+
Compile a regular expression and initialize internal vars.
44
+
"""
45
+
46
+
self.cache = cache
47
+
self.last_match = None
48
+
49
+
self._add_regex(string, flags)
50
+
51
+
def __str__(self):
52
+
"""
53
+
Return the regular expression pattern.
54
+
"""
55
+
return self.regex.pattern
56
+
57
+
def __add__(self, other):
58
+
"""
59
+
Allows adding two regular expressions into one.
60
+
"""
61
+
62
+
return Re(str(self) + str(other), cache=self.cache or other.cache,
63
+
flags=self.regex.flags | other.regex.flags)
64
+
65
+
def match(self, string):
66
+
"""
67
+
Handles a re.match storing its results
68
+
"""
69
+
70
+
self.last_match = self.regex.match(string)
71
+
return self.last_match
72
+
73
+
def search(self, string):
74
+
"""
75
+
Handles a re.search storing its results
76
+
"""
77
+
78
+
self.last_match = self.regex.search(string)
79
+
return self.last_match
80
+
81
+
def findall(self, string):
82
+
"""
83
+
Alias to re.findall
84
+
"""
85
+
86
+
return self.regex.findall(string)
87
+
88
+
def split(self, string):
89
+
"""
90
+
Alias to re.split
91
+
"""
92
+
93
+
return self.regex.split(string)
94
+
95
+
def sub(self, sub, string, count=0):
96
+
"""
97
+
Alias to re.sub
98
+
"""
99
+
100
+
return self.regex.sub(sub, string, count=count)
101
+
102
+
def group(self, num):
103
+
"""
104
+
Returns the group results of the last match
105
+
"""
106
+
107
+
return self.last_match.group(num)
108
+
109
+
110
+
class NestedMatch:
111
+
"""
112
+
Finding nested delimiters is hard with regular expressions. It is
113
+
even harder on Python with its normal re module, as there are several
114
+
advanced regular expressions that are missing.
115
+
116
+
This is the case of this pattern:
117
+
118
+
'\\bSTRUCT_GROUP(\\(((?:(?>[^)(]+)|(?1))*)\\))[^;]*;'
119
+
120
+
which is used to properly match open/close parenthesis of the
121
+
string search STRUCT_GROUP(),
122
+
123
+
Add a class that counts pairs of delimiters, using it to match and
124
+
replace nested expressions.
125
+
126
+
The original approach was suggested by:
127
+
https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex
128
+
129
+
Although I re-implemented it to make it more generic and match 3 types
130
+
of delimiters. The logic checks if delimiters are paired. If not, it
131
+
will ignore the search string.
132
+
"""
133
+
134
+
# TODO:
135
+
# Right now, regular expressions to match it are defined only up to
136
+
# the start delimiter, e.g.:
137
+
#
138
+
# \bSTRUCT_GROUP\(
139
+
#
140
+
# is similar to: STRUCT_GROUP\((.*)\)
141
+
# except that the content inside the match group is delimiter's aligned.
142
+
#
143
+
# The content inside parenthesis are converted into a single replace
144
+
# group (e.g. r`\1').
145
+
#
146
+
# It would be nice to change such definition to support multiple
147
+
# match groups, allowing a regex equivalent to.
148
+
#
149
+
# FOO\((.*), (.*), (.*)\)
150
+
#
151
+
# it is probably easier to define it not as a regular expression, but
152
+
# with some lexical definition like:
153
+
#
154
+
# FOO(arg1, arg2, arg3)
155
+
156
+
DELIMITER_PAIRS = {
157
+
'{': '}',
158
+
'(': ')',
159
+
'[': ']',
160
+
}
161
+
162
+
RE_DELIM = re.compile(r'[\{\}\[\]\(\)]')
163
+
164
+
def _search(self, regex, line):
165
+
"""
166
+
Finds paired blocks for a regex that ends with a delimiter.
167
+
168
+
The suggestion of using finditer to match pairs came from:
169
+
https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex
170
+
but I ended using a different implementation to align all three types
171
+
of delimiters and seek for an initial regular expression.
172
+
173
+
The algorithm seeks for open/close paired delimiters and place them
174
+
into a stack, yielding a start/stop position of each match when the
175
+
stack is zeroed.
176
+
177
+
The algorithm shoud work fine for properly paired lines, but will
178
+
silently ignore end delimiters that preceeds an start delimiter.
179
+
This should be OK for kernel-doc parser, as unaligned delimiters
180
+
would cause compilation errors. So, we don't need to rise exceptions
181
+
to cover such issues.
182
+
"""
183
+
184
+
stack = []
185
+
186
+
for match_re in regex.finditer(line):
187
+
start = match_re.start()
188
+
offset = match_re.end()
189
+
190
+
d = line[offset - 1]
191
+
if d not in self.DELIMITER_PAIRS:
192
+
continue
193
+
194
+
end = self.DELIMITER_PAIRS[d]
195
+
stack.append(end)
196
+
197
+
for match in self.RE_DELIM.finditer(line[offset:]):
198
+
pos = match.start() + offset
199
+
200
+
d = line[pos]
201
+
202
+
if d in self.DELIMITER_PAIRS:
203
+
end = self.DELIMITER_PAIRS[d]
204
+
205
+
stack.append(end)
206
+
continue
207
+
208
+
# Does the end delimiter match what it is expected?
209
+
if stack and d == stack[-1]:
210
+
stack.pop()
211
+
212
+
if not stack:
213
+
yield start, offset, pos + 1
214
+
break
215
+
216
+
def search(self, regex, line):
217
+
"""
218
+
This is similar to re.search:
219
+
220
+
It matches a regex that it is followed by a delimiter,
221
+
returning occurrences only if all delimiters are paired.
222
+
"""
223
+
224
+
for t in self._search(regex, line):
225
+
226
+
yield line[t[0]:t[2]]
227
+
228
+
def sub(self, regex, sub, line, count=0):
229
+
"""
230
+
This is similar to re.sub:
231
+
232
+
It matches a regex that it is followed by a delimiter,
233
+
replacing occurrences only if all delimiters are paired.
234
+
235
+
if r'\1' is used, it works just like re: it places there the
236
+
matched paired data with the delimiter stripped.
237
+
238
+
If count is different than zero, it will replace at most count
239
+
items.
240
+
"""
241
+
out = ""
242
+
243
+
cur_pos = 0
244
+
n = 0
245
+
246
+
for start, end, pos in self._search(regex, line):
247
+
out += line[cur_pos:start]
248
+
249
+
# Value, ignoring start/end delimiters
250
+
value = line[end:pos - 1]
251
+
252
+
# replaces \1 at the sub string, if \1 is used there
253
+
new_sub = sub
254
+
new_sub = new_sub.replace(r'\1', value)
255
+
256
+
out += new_sub
257
+
258
+
# Drop end ';' if any
259
+
if line[pos] == ';':
260
+
pos += 1
261
+
262
+
cur_pos = pos
263
+
n += 1
264
+
265
+
if count and count >= n:
266
+
break
267
+
268
+
# Append the remaining string
269
+
l = len(line)
270
+
out += line[cur_pos:l]
271
+
272
+
return out