Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
scripts/get_abi.py: add support for undefined ABIs
The undefined logic is complex and has lots of magic on it.
Implement it, using the same algorithm we have at get_abi.pl. Yet,
some tweaks to optimize performance and to make the code simpler
were added here:
- at the perl version, the tree graph had loops, so we had to
use BFS to traverse it. On this version, the graph is a tree,
so, it simplifies the what group for sysfs aliases;
- the logic which splits regular expressions into subgroups
was re-written to make it faster;
- it may optionally use multiple processes to search for symbol
matches;
- it has some additional debug levels.
Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
Signed-off-by: Jonathan Corbet <corbet@lwn.net>
Link: https://lore.kernel.org/r/1529c255845d117696d5af57d8dc05554663afdf.1739182025.git.mchehab+huawei@kernel.org
authored by
Mauro Carvalho Chehab
and committed by
Jonathan Corbet
0d5fd968
6649b421
+693
-3
+68
scripts/get_abi.py
+68
scripts/get_abi.py
···
20
20
sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR))
21
21
22
22
from abi_parser import AbiParser # pylint: disable=C0413
23
+
from abi_regex import AbiRegex # pylint: disable=C0413
23
24
from helpers import ABI_DIR, DEBUG_HELP # pylint: disable=C0413
25
+
from system_symbols import SystemSymbols # pylint: disable=C0413
24
26
25
27
# Command line classes
26
28
···
113
111
parser.parse_abi()
114
112
parser.search_symbols(args.expression)
115
113
114
+
UNDEFINED_DESC="""
115
+
Check undefined ABIs on local machine.
116
+
117
+
Read sysfs devnodes and check if the devnodes there are defined inside
118
+
ABI documentation.
119
+
120
+
The search logic tries to minimize the number of regular expressions to
121
+
search per each symbol.
122
+
123
+
By default, it runs on a single CPU, as Python support for CPU threads
124
+
is still experimental, and multi-process runs on Python is very slow.
125
+
126
+
On experimental tests, if the number of ABI symbols to search per devnode
127
+
is contained on a limit of ~150 regular expressions, using a single CPU
128
+
is a lot faster than using multiple processes. However, if the number of
129
+
regular expressions to check is at the order of ~30000, using multiple
130
+
CPUs speeds up the check.
131
+
"""
132
+
133
+
class AbiUndefined:
134
+
"""
135
+
Initialize an argparse subparser for logic to check undefined ABI at
136
+
the current machine's sysfs
137
+
"""
138
+
139
+
def __init__(self, subparsers):
140
+
"""Initialize argparse subparsers"""
141
+
142
+
parser = subparsers.add_parser("undefined",
143
+
formatter_class=argparse.RawTextHelpFormatter,
144
+
description=UNDEFINED_DESC)
145
+
146
+
parser.add_argument("-S", "--sysfs-dir", default="/sys",
147
+
help="directory where sysfs is mounted")
148
+
parser.add_argument("-s", "--search-string",
149
+
help="search string regular expression to limit symbol search")
150
+
parser.add_argument("-H", "--show-hints", action="store_true",
151
+
help="Hints about definitions for missing ABI symbols.")
152
+
parser.add_argument("-j", "--jobs", "--max-workers", type=int, default=1,
153
+
help="If bigger than one, enables multiprocessing.")
154
+
parser.add_argument("-c", "--max-chunk-size", type=int, default=50,
155
+
help="Maximum number of chunk size")
156
+
parser.add_argument("-f", "--found", action="store_true",
157
+
help="Also show found items. "
158
+
"Helpful to debug the parser."),
159
+
parser.add_argument("-d", "--dry-run", action="store_true",
160
+
help="Don't actually search for undefined. "
161
+
"Helpful to debug the parser."),
162
+
163
+
parser.set_defaults(func=self.run)
164
+
165
+
def run(self, args):
166
+
"""Run subparser"""
167
+
168
+
abi = AbiRegex(args.dir, debug=args.debug,
169
+
search_string=args.search_string)
170
+
171
+
abi_symbols = SystemSymbols(abi=abi, hints=args.show_hints,
172
+
sysfs=args.sysfs_dir)
173
+
174
+
abi_symbols.check_undefined_symbols(dry_run=args.dry_run,
175
+
found=args.found,
176
+
max_workers=args.jobs,
177
+
chunk_size=args.max_chunk_size)
178
+
116
179
117
180
def main():
118
181
"""Main program"""
···
192
125
AbiRest(subparsers)
193
126
AbiValidate(subparsers)
194
127
AbiSearch(subparsers)
128
+
AbiUndefined(subparsers)
195
129
196
130
args = parser.parse_args()
197
131
+234
scripts/lib/abi/abi_regex.py
+234
scripts/lib/abi/abi_regex.py
···
1
+
#!/usr/bin/env python3
2
+
# xxpylint: disable=R0903
3
+
# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>.
4
+
# SPDX-License-Identifier: GPL-2.0
5
+
6
+
"""
7
+
Convert ABI what into regular expressions
8
+
"""
9
+
10
+
import re
11
+
import sys
12
+
13
+
from pprint import pformat
14
+
15
+
from abi_parser import AbiParser
16
+
from helpers import AbiDebug
17
+
18
+
class AbiRegex(AbiParser):
19
+
"""Extends AbiParser to search ABI nodes with regular expressions"""
20
+
21
+
# Escape only ASCII visible characters
22
+
escape_symbols = r"([\x21-\x29\x2b-\x2d\x3a-\x40\x5c\x60\x7b-\x7e])"
23
+
leave_others = "others"
24
+
25
+
# Tuples with regular expressions to be compiled and replacement data
26
+
re_whats = [
27
+
# Drop escape characters that might exist
28
+
(re.compile("\\\\"), ""),
29
+
30
+
# Temporarily escape dot characters
31
+
(re.compile(r"\."), "\xf6"),
32
+
33
+
# Temporarily change [0-9]+ type of patterns
34
+
(re.compile(r"\[0\-9\]\+"), "\xff"),
35
+
36
+
# Temporarily change [\d+-\d+] type of patterns
37
+
(re.compile(r"\[0\-\d+\]"), "\xff"),
38
+
(re.compile(r"\[0:\d+\]"), "\xff"),
39
+
(re.compile(r"\[(\d+)\]"), "\xf4\\\\d+\xf5"),
40
+
41
+
# Temporarily change [0-9] type of patterns
42
+
(re.compile(r"\[(\d)\-(\d)\]"), "\xf4\1-\2\xf5"),
43
+
44
+
# Handle multiple option patterns
45
+
(re.compile(r"[\{\<\[]([\w_]+)(?:[,|]+([\w_]+)){1,}[\}\>\]]"), r"(\1|\2)"),
46
+
47
+
# Handle wildcards
48
+
(re.compile(r"([^\/])\*"), "\\1\\\\w\xf7"),
49
+
(re.compile(r"/\*/"), "/.*/"),
50
+
(re.compile(r"/\xf6\xf6\xf6"), "/.*"),
51
+
(re.compile(r"\<[^\>]+\>"), "\\\\w\xf7"),
52
+
(re.compile(r"\{[^\}]+\}"), "\\\\w\xf7"),
53
+
(re.compile(r"\[[^\]]+\]"), "\\\\w\xf7"),
54
+
55
+
(re.compile(r"XX+"), "\\\\w\xf7"),
56
+
(re.compile(r"([^A-Z])[XYZ]([^A-Z])"), "\\1\\\\w\xf7\\2"),
57
+
(re.compile(r"([^A-Z])[XYZ]$"), "\\1\\\\w\xf7"),
58
+
(re.compile(r"_[AB]_"), "_\\\\w\xf7_"),
59
+
60
+
# Recover [0-9] type of patterns
61
+
(re.compile(r"\xf4"), "["),
62
+
(re.compile(r"\xf5"), "]"),
63
+
64
+
# Remove duplicated spaces
65
+
(re.compile(r"\s+"), r" "),
66
+
67
+
# Special case: drop comparison as in:
68
+
# What: foo = <something>
69
+
# (this happens on a few IIO definitions)
70
+
(re.compile(r"\s*\=.*$"), ""),
71
+
72
+
# Escape all other symbols
73
+
(re.compile(escape_symbols), r"\\\1"),
74
+
(re.compile(r"\\\\"), r"\\"),
75
+
(re.compile(r"\\([\[\]\(\)\|])"), r"\1"),
76
+
(re.compile(r"(\d+)\\(-\d+)"), r"\1\2"),
77
+
78
+
(re.compile(r"\xff"), r"\\d+"),
79
+
80
+
# Special case: IIO ABI which a parenthesis.
81
+
(re.compile(r"sqrt(.*)"), r"sqrt(.*)"),
82
+
83
+
# Simplify regexes with multiple .*
84
+
(re.compile(r"(?:\.\*){2,}"), ""),
85
+
86
+
# Recover dot characters
87
+
(re.compile(r"\xf6"), "\\."),
88
+
# Recover plus characters
89
+
(re.compile(r"\xf7"), "+"),
90
+
]
91
+
re_has_num = re.compile(r"\\d")
92
+
93
+
# Symbol name after escape_chars that are considered a devnode basename
94
+
re_symbol_name = re.compile(r"(\w|\\[\.\-\:])+$")
95
+
96
+
# List of popular group names to be skipped to minimize regex group size
97
+
# Use AbiDebug.SUBGROUP_SIZE to detect those
98
+
skip_names = set(["devices", "hwmon"])
99
+
100
+
def regex_append(self, what, new):
101
+
"""
102
+
Get a search group for a subset of regular expressions.
103
+
104
+
As ABI may have thousands of symbols, using a for to search all
105
+
regular expressions is at least O(n^2). When there are wildcards,
106
+
the complexity increases substantially, eventually becoming exponential.
107
+
108
+
To avoid spending too much time on them, use a logic to split
109
+
them into groups. The smaller the group, the better, as it would
110
+
mean that searches will be confined to a small number of regular
111
+
expressions.
112
+
113
+
The conversion to a regex subset is tricky, as we need something
114
+
that can be easily obtained from the sysfs symbol and from the
115
+
regular expression. So, we need to discard nodes that have
116
+
wildcards.
117
+
118
+
If it can't obtain a subgroup, place the regular expression inside
119
+
a special group (self.leave_others).
120
+
"""
121
+
122
+
search_group = None
123
+
124
+
for search_group in reversed(new.split("/")):
125
+
if not search_group or search_group in self.skip_names:
126
+
continue
127
+
if self.re_symbol_name.match(search_group):
128
+
break
129
+
130
+
if not search_group:
131
+
search_group = self.leave_others
132
+
133
+
if self.debug & AbiDebug.SUBGROUP_MAP:
134
+
self.log.debug("%s: mapped as %s", what, search_group)
135
+
136
+
try:
137
+
if search_group not in self.regex_group:
138
+
self.regex_group[search_group] = []
139
+
140
+
self.regex_group[search_group].append(re.compile(new))
141
+
if self.search_string:
142
+
if what.find(self.search_string) >= 0:
143
+
print(f"What: {what}")
144
+
except re.PatternError:
145
+
self.log.warning("Ignoring '%s' as it produced an invalid regex:\n"
146
+
" '%s'", what, new)
147
+
148
+
def get_regexes(self, what):
149
+
"""
150
+
Given an ABI devnode, return a list of all regular expressions that
151
+
may match it, based on the sub-groups created by regex_append()
152
+
"""
153
+
154
+
re_list = []
155
+
156
+
patches = what.split("/")
157
+
patches.reverse()
158
+
patches.append(self.leave_others)
159
+
160
+
for search_group in patches:
161
+
if search_group in self.regex_group:
162
+
re_list += self.regex_group[search_group]
163
+
164
+
return re_list
165
+
166
+
def __init__(self, *args, **kwargs):
167
+
"""
168
+
Override init method to get verbose argument
169
+
"""
170
+
171
+
self.regex_group = None
172
+
self.search_string = None
173
+
self.re_string = None
174
+
175
+
if "search_string" in kwargs:
176
+
self.search_string = kwargs.get("search_string")
177
+
del kwargs["search_string"]
178
+
179
+
if self.search_string:
180
+
181
+
try:
182
+
self.re_string = re.compile(self.search_string)
183
+
except re.PatternError as e:
184
+
msg = f"{self.search_string} is not a valid regular expression"
185
+
raise ValueError(msg) from e
186
+
187
+
super().__init__(*args, **kwargs)
188
+
189
+
def parse_abi(self, *args, **kwargs):
190
+
191
+
super().parse_abi(*args, **kwargs)
192
+
193
+
self.regex_group = {}
194
+
195
+
print("Converting ABI What fields into regexes...", file=sys.stderr)
196
+
197
+
for t in sorted(self.data.items(), key=lambda x: x[0]):
198
+
v = t[1]
199
+
if v.get("type") == "File":
200
+
continue
201
+
202
+
v["regex"] = []
203
+
204
+
for what in v.get("what", []):
205
+
if not what.startswith("/sys"):
206
+
continue
207
+
208
+
new = what
209
+
for r, s in self.re_whats:
210
+
try:
211
+
new = r.sub(s, new)
212
+
except re.PatternError as e:
213
+
# Help debugging troubles with new regexes
214
+
raise re.PatternError(f"{e}\nwhile re.sub('{r.pattern}', {s}, str)") from e
215
+
216
+
v["regex"].append(new)
217
+
218
+
if self.debug & AbiDebug.REGEX:
219
+
self.log.debug("%-90s <== %s", new, what)
220
+
221
+
# Store regex into a subgroup to speedup searches
222
+
self.regex_append(what, new)
223
+
224
+
if self.debug & AbiDebug.SUBGROUP_DICT:
225
+
self.log.debug("%s", pformat(self.regex_group))
226
+
227
+
if self.debug & AbiDebug.SUBGROUP_SIZE:
228
+
biggestd_keys = sorted(self.regex_group.keys(),
229
+
key= lambda k: len(self.regex_group[k]),
230
+
reverse=True)
231
+
232
+
print("Top regex subgroups:", file=sys.stderr)
233
+
for k in biggestd_keys[:10]:
234
+
print(f"{k} has {len(self.regex_group[k])} elements", file=sys.stderr)
+13
-3
scripts/lib/abi/helpers.py
+13
-3
scripts/lib/abi/helpers.py
···
16
16
WHAT_PARSING = 1
17
17
WHAT_OPEN = 2
18
18
DUMP_ABI_STRUCTS = 4
19
+
UNDEFINED = 8
20
+
REGEX = 16
21
+
SUBGROUP_MAP = 32
22
+
SUBGROUP_DICT = 64
23
+
SUBGROUP_SIZE = 128
24
+
GRAPH = 256
19
25
20
26
21
27
DEBUG_HELP = """
22
-
Print debug information according with the level(s),
23
-
which is given by the following bitmask:
24
-
25
28
1 - enable debug parsing logic
26
29
2 - enable debug messages on file open
27
30
4 - enable debug for ABI parse data
31
+
8 - enable extra debug information to identify troubles
32
+
with ABI symbols found at the local machine that
33
+
weren't found on ABI documentation (used only for
34
+
undefined subcommand)
35
+
16 - enable debug for what to regex conversion
36
+
32 - enable debug for symbol regex subgroups
37
+
64 - enable debug for sysfs graph tree variable
28
38
"""
+378
scripts/lib/abi/system_symbols.py
+378
scripts/lib/abi/system_symbols.py
···
1
+
#!/usr/bin/env python3
2
+
# pylint: disable=R0902,R0912,R0914,R0915,R1702
3
+
# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>.
4
+
# SPDX-License-Identifier: GPL-2.0
5
+
6
+
"""
7
+
Parse ABI documentation and produce results from it.
8
+
"""
9
+
10
+
import os
11
+
import re
12
+
import sys
13
+
14
+
from concurrent import futures
15
+
from datetime import datetime
16
+
from random import shuffle
17
+
18
+
from helpers import AbiDebug
19
+
20
+
class SystemSymbols:
21
+
"""Stores arguments for the class and initialize class vars"""
22
+
23
+
def graph_add_file(self, path, link=None):
24
+
"""
25
+
add a file path to the sysfs graph stored at self.root
26
+
"""
27
+
28
+
if path in self.files:
29
+
return
30
+
31
+
name = ""
32
+
ref = self.root
33
+
for edge in path.split("/"):
34
+
name += edge + "/"
35
+
if edge not in ref:
36
+
ref[edge] = {"__name": [name.rstrip("/")]}
37
+
38
+
ref = ref[edge]
39
+
40
+
if link and link not in ref["__name"]:
41
+
ref["__name"].append(link.rstrip("/"))
42
+
43
+
self.files.add(path)
44
+
45
+
def print_graph(self, root_prefix="", root=None, level=0):
46
+
"""Prints a reference tree graph using UTF-8 characters"""
47
+
48
+
if not root:
49
+
root = self.root
50
+
level = 0
51
+
52
+
# Prevent endless traverse
53
+
if level > 5:
54
+
return
55
+
56
+
if level > 0:
57
+
prefix = "├──"
58
+
last_prefix = "└──"
59
+
else:
60
+
prefix = ""
61
+
last_prefix = ""
62
+
63
+
items = list(root.items())
64
+
65
+
names = root.get("__name", [])
66
+
for k, edge in items:
67
+
if k == "__name":
68
+
continue
69
+
70
+
if not k:
71
+
k = "/"
72
+
73
+
if len(names) > 1:
74
+
k += " links: " + ",".join(names[1:])
75
+
76
+
if edge == items[-1][1]:
77
+
print(root_prefix + last_prefix + k)
78
+
p = root_prefix
79
+
if level > 0:
80
+
p += " "
81
+
self.print_graph(p, edge, level + 1)
82
+
else:
83
+
print(root_prefix + prefix + k)
84
+
p = root_prefix + "│ "
85
+
self.print_graph(p, edge, level + 1)
86
+
87
+
def _walk(self, root):
88
+
"""
89
+
Walk through sysfs to get all devnodes that aren't ignored.
90
+
91
+
By default, uses /sys as sysfs mounting point. If another
92
+
directory is used, it replaces them to /sys at the patches.
93
+
"""
94
+
95
+
with os.scandir(root) as obj:
96
+
for entry in obj:
97
+
path = os.path.join(root, entry.name)
98
+
if self.sysfs:
99
+
p = path.replace(self.sysfs, "/sys", count=1)
100
+
else:
101
+
p = path
102
+
103
+
if self.re_ignore.search(p):
104
+
return
105
+
106
+
# Handle link first to avoid directory recursion
107
+
if entry.is_symlink():
108
+
real = os.path.realpath(path)
109
+
if not self.sysfs:
110
+
self.aliases[path] = real
111
+
else:
112
+
real = real.replace(self.sysfs, "/sys", count=1)
113
+
114
+
# Add absfile location to graph if it doesn't exist
115
+
if not self.re_ignore.search(real):
116
+
# Add link to the graph
117
+
self.graph_add_file(real, p)
118
+
119
+
elif entry.is_file():
120
+
self.graph_add_file(p)
121
+
122
+
elif entry.is_dir():
123
+
self._walk(path)
124
+
125
+
def __init__(self, abi, sysfs="/sys", hints=False):
126
+
"""
127
+
Initialize internal variables and get a list of all files inside
128
+
sysfs that can currently be parsed.
129
+
130
+
Please notice that there are several entries on sysfs that aren't
131
+
documented as ABI. Ignore those.
132
+
133
+
The real paths will be stored under self.files. Aliases will be
134
+
stored in separate, as self.aliases.
135
+
"""
136
+
137
+
self.abi = abi
138
+
self.log = abi.log
139
+
140
+
if sysfs != "/sys":
141
+
self.sysfs = sysfs.rstrip("/")
142
+
else:
143
+
self.sysfs = None
144
+
145
+
self.hints = hints
146
+
147
+
self.root = {}
148
+
self.aliases = {}
149
+
self.files = set()
150
+
151
+
dont_walk = [
152
+
# Those require root access and aren't documented at ABI
153
+
f"^{sysfs}/kernel/debug",
154
+
f"^{sysfs}/kernel/tracing",
155
+
f"^{sysfs}/fs/pstore",
156
+
f"^{sysfs}/fs/bpf",
157
+
f"^{sysfs}/fs/fuse",
158
+
159
+
# This is not documented at ABI
160
+
f"^{sysfs}/module",
161
+
162
+
f"^{sysfs}/fs/cgroup", # this is big and has zero docs under ABI
163
+
f"^{sysfs}/firmware", # documented elsewhere: ACPI, DT bindings
164
+
"sections|notes", # aren't actually part of ABI
165
+
166
+
# kernel-parameters.txt - not easy to parse
167
+
"parameters",
168
+
]
169
+
170
+
self.re_ignore = re.compile("|".join(dont_walk))
171
+
172
+
print(f"Reading {sysfs} directory contents...", file=sys.stderr)
173
+
self._walk(sysfs)
174
+
175
+
def check_file(self, refs, found):
176
+
"""Check missing ABI symbols for a given sysfs file"""
177
+
178
+
res_list = []
179
+
180
+
try:
181
+
for names in refs:
182
+
fname = names[0]
183
+
184
+
res = {
185
+
"found": False,
186
+
"fname": fname,
187
+
"msg": "",
188
+
}
189
+
res_list.append(res)
190
+
191
+
re_what = self.abi.get_regexes(fname)
192
+
if not re_what:
193
+
self.abi.log.warning(f"missing rules for {fname}")
194
+
continue
195
+
196
+
for name in names:
197
+
for r in re_what:
198
+
if self.abi.debug & AbiDebug.UNDEFINED:
199
+
self.log.debug("check if %s matches '%s'", name, r.pattern)
200
+
if r.match(name):
201
+
res["found"] = True
202
+
if found:
203
+
res["msg"] += f" {fname}: regex:\n\t"
204
+
continue
205
+
206
+
if self.hints and not res["found"]:
207
+
res["msg"] += f" {fname} not found. Tested regexes:\n"
208
+
for r in re_what:
209
+
res["msg"] += " " + r.pattern + "\n"
210
+
211
+
except KeyboardInterrupt:
212
+
pass
213
+
214
+
return res_list
215
+
216
+
def _ref_interactor(self, root):
217
+
"""Recursive function to interact over the sysfs tree"""
218
+
219
+
for k, v in root.items():
220
+
if isinstance(v, dict):
221
+
yield from self._ref_interactor(v)
222
+
223
+
if root == self.root or k == "__name":
224
+
continue
225
+
226
+
if self.abi.re_string:
227
+
fname = v["__name"][0]
228
+
if self.abi.re_string.search(fname):
229
+
yield v
230
+
else:
231
+
yield v
232
+
233
+
234
+
def get_fileref(self, all_refs, chunk_size):
235
+
"""Interactor to group refs into chunks"""
236
+
237
+
n = 0
238
+
refs = []
239
+
240
+
for ref in all_refs:
241
+
refs.append(ref)
242
+
243
+
n += 1
244
+
if n >= chunk_size:
245
+
yield refs
246
+
n = 0
247
+
refs = []
248
+
249
+
yield refs
250
+
251
+
def check_undefined_symbols(self, max_workers=None, chunk_size=50,
252
+
found=None, dry_run=None):
253
+
"""Seach ABI for sysfs symbols missing documentation"""
254
+
255
+
self.abi.parse_abi()
256
+
257
+
if self.abi.debug & AbiDebug.GRAPH:
258
+
self.print_graph()
259
+
260
+
all_refs = []
261
+
for ref in self._ref_interactor(self.root):
262
+
all_refs.append(ref["__name"])
263
+
264
+
if dry_run:
265
+
print("Would check", file=sys.stderr)
266
+
for ref in all_refs:
267
+
print(", ".join(ref))
268
+
269
+
return
270
+
271
+
print("Starting to search symbols (it may take several minutes):",
272
+
file=sys.stderr)
273
+
start = datetime.now()
274
+
old_elapsed = None
275
+
276
+
# Python doesn't support multithreading due to limitations on its
277
+
# global lock (GIL). While Python 3.13 finally made GIL optional,
278
+
# there are still issues related to it. Also, we want to have
279
+
# backward compatibility with older versions of Python.
280
+
#
281
+
# So, use instead multiprocess. However, Python is very slow passing
282
+
# data from/to multiple processes. Also, it may consume lots of memory
283
+
# if the data to be shared is not small. So, we need to group workload
284
+
# in chunks that are big enough to generate performance gains while
285
+
# not being so big that would cause out-of-memory.
286
+
287
+
num_refs = len(all_refs)
288
+
print(f"Number of references to parse: {num_refs}", file=sys.stderr)
289
+
290
+
if not max_workers:
291
+
max_workers = os.cpu_count()
292
+
elif max_workers > os.cpu_count():
293
+
max_workers = os.cpu_count()
294
+
295
+
max_workers = max(max_workers, 1)
296
+
297
+
max_chunk_size = int((num_refs + max_workers - 1) / max_workers)
298
+
chunk_size = min(chunk_size, max_chunk_size)
299
+
chunk_size = max(1, chunk_size)
300
+
301
+
if max_workers > 1:
302
+
executor = futures.ProcessPoolExecutor
303
+
304
+
# Place references in a random order. This may help improving
305
+
# performance, by mixing complex/simple expressions when creating
306
+
# chunks
307
+
shuffle(all_refs)
308
+
else:
309
+
# Python has a high overhead with processes. When there's just
310
+
# one worker, it is faster to not create a new process.
311
+
# Yet, User still deserves to have a progress print. So, use
312
+
# python's "thread", which is actually a single process, using
313
+
# an internal schedule to switch between tasks. No performance
314
+
# gains for non-IO tasks, but still it can be quickly interrupted
315
+
# from time to time to display progress.
316
+
executor = futures.ThreadPoolExecutor
317
+
318
+
not_found = []
319
+
f_list = []
320
+
with executor(max_workers=max_workers) as exe:
321
+
for refs in self.get_fileref(all_refs, chunk_size):
322
+
if refs:
323
+
try:
324
+
f_list.append(exe.submit(self.check_file, refs, found))
325
+
326
+
except KeyboardInterrupt:
327
+
return
328
+
329
+
total = len(f_list)
330
+
331
+
if not total:
332
+
if self.abi.re_string:
333
+
print(f"No ABI symbol matches {self.abi.search_string}")
334
+
else:
335
+
self.abi.log.warning("No ABI symbols found")
336
+
return
337
+
338
+
print(f"{len(f_list):6d} jobs queued on {max_workers} workers",
339
+
file=sys.stderr)
340
+
341
+
while f_list:
342
+
try:
343
+
t = futures.wait(f_list, timeout=1,
344
+
return_when=futures.FIRST_COMPLETED)
345
+
346
+
done = t[0]
347
+
348
+
for fut in done:
349
+
res_list = fut.result()
350
+
351
+
for res in res_list:
352
+
if not res["found"]:
353
+
not_found.append(res["fname"])
354
+
if res["msg"]:
355
+
print(res["msg"])
356
+
357
+
f_list.remove(fut)
358
+
except KeyboardInterrupt:
359
+
return
360
+
361
+
except RuntimeError as e:
362
+
self.abi.log.warning(f"Future: {e}")
363
+
break
364
+
365
+
if sys.stderr.isatty():
366
+
elapsed = str(datetime.now() - start).split(".", maxsplit=1)[0]
367
+
if len(f_list) < total:
368
+
elapsed += f" ({total - len(f_list)}/{total} jobs completed). "
369
+
if elapsed != old_elapsed:
370
+
print(elapsed + "\r", end="", flush=True,
371
+
file=sys.stderr)
372
+
old_elapsed = elapsed
373
+
374
+
elapsed = str(datetime.now() - start).split(".", maxsplit=1)[0]
375
+
print(elapsed, file=sys.stderr)
376
+
377
+
for f in sorted(not_found):
378
+
print(f"{f} not found.")