Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
scripts: test_doc_build.py: make capture assynchronous
Prepare the tool to allow writing the output into log files.
For such purpose, receive stdin/stdout messages asynchronously.
Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
Signed-off-by: Jonathan Corbet <corbet@lwn.net>
Link: https://lore.kernel.org/r/9b0a60b5047137b5ba764701268da992767b128c.1750571906.git.mchehab+huawei@kernel.org
authored by
Mauro Carvalho Chehab
and committed by
Jonathan Corbet
7649db7d
54c147f4
+244
-127
+244
-127
scripts/test_doc_build.py
+244
-127
scripts/test_doc_build.py
···
2
2
# SPDX-License-Identifier: GPL-2.0
3
3
# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
4
4
#
5
-
# pylint: disable=C0103,R1715
5
+
# pylint: disable=R0903,R0913,R0914,R0917
6
6
7
7
"""
8
8
Install minimal supported requirements for different Sphinx versions
···
10
10
"""
11
11
12
12
import argparse
13
+
import asyncio
13
14
import os.path
14
15
import sys
15
16
import time
16
-
17
-
from subprocess import run
17
+
import subprocess
18
18
19
19
# Minimal python version supported by the building system
20
-
python_bin = "python3.9"
20
+
MINIMAL_PYTHON_VERSION = "python3.9"
21
21
22
22
# Starting from 8.0.2, Python 3.9 becomes too old
23
-
python_changes = {(8, 0, 2): "python3"}
23
+
PYTHON_VER_CHANGES = {(8, 0, 2): "python3"}
24
24
25
25
# Sphinx versions to be installed and their incremental requirements
26
-
sphinx_requirements = {
26
+
SPHINX_REQUIREMENTS = {
27
27
(3, 4, 3): {
28
28
"alabaster": "0.7.13",
29
29
"babel": "2.17.0",
···
101
101
}
102
102
103
103
104
+
class AsyncCommands:
105
+
"""Excecute command synchronously"""
106
+
107
+
stdout = None
108
+
stderr = None
109
+
output = None
110
+
111
+
async def _read(self, stream, verbose, is_info):
112
+
"""Ancillary routine to capture while displaying"""
113
+
114
+
while stream is not None:
115
+
line = await stream.readline()
116
+
if line:
117
+
out = line.decode("utf-8", errors="backslashreplace")
118
+
self.output += out
119
+
if is_info:
120
+
if verbose:
121
+
print(out.rstrip("\n"))
122
+
123
+
self.stdout += out
124
+
else:
125
+
if verbose:
126
+
print(out.rstrip("\n"), file=sys.stderr)
127
+
128
+
self.stderr += out
129
+
else:
130
+
break
131
+
132
+
async def run(self, cmd, capture_output=False, check=False,
133
+
env=None, verbose=True):
134
+
135
+
"""
136
+
Execute an arbitrary command, handling errors.
137
+
138
+
Please notice that this class is not thread safe
139
+
"""
140
+
141
+
self.stdout = ""
142
+
self.stderr = ""
143
+
self.output = ""
144
+
145
+
if verbose:
146
+
print("$ ", " ".join(cmd))
147
+
148
+
proc = await asyncio.create_subprocess_exec(cmd[0],
149
+
*cmd[1:],
150
+
env=env,
151
+
stdout=asyncio.subprocess.PIPE,
152
+
stderr=asyncio.subprocess.PIPE)
153
+
154
+
# Handle input and output in realtime
155
+
await asyncio.gather(
156
+
self._read(proc.stdout, verbose, True),
157
+
self._read(proc.stderr, verbose, False),
158
+
)
159
+
160
+
await proc.wait()
161
+
162
+
if check and proc.returncode > 0:
163
+
raise subprocess.CalledProcessError(returncode=proc.returncode,
164
+
cmd=" ".join(cmd),
165
+
output=self.stdout,
166
+
stderr=self.stderr)
167
+
168
+
if capture_output:
169
+
if proc.returncode > 0:
170
+
print("Error {proc.returncode}", file=sys.stderr)
171
+
return ""
172
+
173
+
return self.output
174
+
175
+
ret = subprocess.CompletedProcess(args=cmd,
176
+
returncode=proc.returncode,
177
+
stdout=self.stdout,
178
+
stderr=self.stderr)
179
+
180
+
return ret
181
+
182
+
183
+
class SphinxVenv:
184
+
"""
185
+
Installs Sphinx on one virtual env per Sphinx version with a minimal
186
+
set of dependencies, adjusting them to each specific version.
187
+
"""
188
+
189
+
def __init__(self):
190
+
"""Initialize instance variables"""
191
+
192
+
self.built_time = {}
193
+
self.first_run = True
194
+
195
+
async def _handle_version(self, args, cur_ver, cur_requirements, python_bin):
196
+
"""Handle a single Sphinx version"""
197
+
198
+
cmd = AsyncCommands()
199
+
200
+
ver = ".".join(map(str, cur_ver))
201
+
202
+
if not self.first_run and args.wait_input and args.make:
203
+
ret = input("Press Enter to continue or 'a' to abort: ").strip().lower()
204
+
if ret == "a":
205
+
print("Aborted.")
206
+
sys.exit()
207
+
else:
208
+
self.first_run = False
209
+
210
+
venv_dir = f"Sphinx_{ver}"
211
+
req_file = f"requirements_{ver}.txt"
212
+
213
+
print(f"\nSphinx {ver} with {python_bin}")
214
+
215
+
# Create venv
216
+
await cmd.run([python_bin, "-m", "venv", venv_dir], check=True)
217
+
pip = os.path.join(venv_dir, "bin/pip")
218
+
219
+
# Create install list
220
+
reqs = []
221
+
for pkg, verstr in cur_requirements.items():
222
+
reqs.append(f"{pkg}=={verstr}")
223
+
224
+
reqs.append(f"Sphinx=={ver}")
225
+
226
+
await cmd.run([pip, "install"] + reqs, check=True, verbose=True)
227
+
228
+
# Freeze environment
229
+
result = await cmd.run([pip, "freeze"], verbose=False, check=True)
230
+
231
+
# Pip install succeeded. Write requirements file
232
+
if args.write:
233
+
with open(req_file, "w", encoding="utf-8") as fp:
234
+
fp.write(result.stdout)
235
+
236
+
if args.make:
237
+
start_time = time.time()
238
+
239
+
# Prepare a venv environment
240
+
env = os.environ.copy()
241
+
bin_dir = os.path.join(venv_dir, "bin")
242
+
env["PATH"] = bin_dir + ":" + env["PATH"]
243
+
env["VIRTUAL_ENV"] = venv_dir
244
+
if "PYTHONHOME" in env:
245
+
del env["PYTHONHOME"]
246
+
247
+
# Test doc build
248
+
await cmd.run(["make", "cleandocs"], env=env, check=True)
249
+
make = ["make"] + args.make_args + ["htmldocs"]
250
+
251
+
print(f". {bin_dir}/activate")
252
+
print(" ".join(make))
253
+
print("deactivate")
254
+
await cmd.run(make, env=env, check=True)
255
+
256
+
end_time = time.time()
257
+
elapsed_time = end_time - start_time
258
+
hours, minutes = divmod(elapsed_time, 3600)
259
+
minutes, seconds = divmod(minutes, 60)
260
+
261
+
hours = int(hours)
262
+
minutes = int(minutes)
263
+
seconds = int(seconds)
264
+
265
+
self.built_time[ver] = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
266
+
267
+
print(f"Finished doc build for Sphinx {ver}. Elapsed time: {self.built_time[ver]}")
268
+
269
+
async def run(self, args):
270
+
"""
271
+
Navigate though multiple Sphinx versions, handling each of them
272
+
on a loop.
273
+
"""
274
+
275
+
cur_requirements = {}
276
+
python_bin = MINIMAL_PYTHON_VERSION
277
+
278
+
for cur_ver, new_reqs in SPHINX_REQUIREMENTS.items():
279
+
cur_requirements.update(new_reqs)
280
+
281
+
if cur_ver in PYTHON_VER_CHANGES: # pylint: disable=R1715
282
+
283
+
python_bin = PYTHON_VER_CHANGES[cur_ver]
284
+
285
+
if args.min_version:
286
+
if cur_ver < args.min_version:
287
+
continue
288
+
289
+
if args.max_version:
290
+
if cur_ver > args.max_version:
291
+
break
292
+
293
+
await self._handle_version(args, cur_ver, cur_requirements,
294
+
python_bin)
295
+
296
+
if args.make:
297
+
print()
298
+
print("Summary:")
299
+
for ver, elapsed_time in sorted(self.built_time.items()):
300
+
print(f"\tSphinx {ver} elapsed time: {elapsed_time}")
301
+
302
+
104
303
def parse_version(ver_str):
105
304
"""Convert a version string into a tuple."""
106
305
107
306
return tuple(map(int, ver_str.split(".")))
108
307
109
308
110
-
parser = argparse.ArgumentParser(description="Build docs for different sphinx_versions.")
309
+
async def main():
310
+
"""Main program"""
111
311
112
-
parser.add_argument('-v', '--version', help='Sphinx single version',
113
-
type=parse_version)
114
-
parser.add_argument('--min-version', "--min", help='Sphinx minimal version',
115
-
type=parse_version)
116
-
parser.add_argument('--max-version', "--max", help='Sphinx maximum version',
117
-
type=parse_version)
118
-
parser.add_argument('-a', '--make_args',
119
-
help='extra arguments for make htmldocs, like SPHINXDIRS=netlink/specs',
120
-
nargs="*")
121
-
parser.add_argument('-w', '--write', help='write a requirements.txt file',
122
-
action='store_true')
123
-
parser.add_argument('-m', '--make',
124
-
help='Make documentation',
125
-
action='store_true')
126
-
parser.add_argument('-i', '--wait-input',
127
-
help='Wait for an enter before going to the next version',
128
-
action='store_true')
312
+
parser = argparse.ArgumentParser(description="Build docs for different sphinx_versions.")
129
313
130
-
args = parser.parse_args()
314
+
parser.add_argument('-v', '--version', help='Sphinx single version',
315
+
type=parse_version)
316
+
parser.add_argument('--min-version', "--min", help='Sphinx minimal version',
317
+
type=parse_version)
318
+
parser.add_argument('--max-version', "--max", help='Sphinx maximum version',
319
+
type=parse_version)
320
+
parser.add_argument('-a', '--make_args',
321
+
help='extra arguments for make htmldocs, like SPHINXDIRS=netlink/specs',
322
+
nargs="*")
323
+
parser.add_argument('-w', '--write', help='write a requirements.txt file',
324
+
action='store_true')
325
+
parser.add_argument('-m', '--make',
326
+
help='Make documentation',
327
+
action='store_true')
328
+
parser.add_argument('-i', '--wait-input',
329
+
help='Wait for an enter before going to the next version',
330
+
action='store_true')
131
331
132
-
if not args.make_args:
133
-
args.make_args = []
332
+
args = parser.parse_args()
134
333
135
-
if args.version:
136
-
if args.min_version or args.max_version:
137
-
sys.exit("Use either --version or --min-version/--max-version")
138
-
else:
139
-
args.min_version = args.version
140
-
args.max_version = args.version
334
+
if not args.make_args:
335
+
args.make_args = []
141
336
142
-
sphinx_versions = sorted(list(sphinx_requirements.keys()))
337
+
if args.version:
338
+
if args.min_version or args.max_version:
339
+
sys.exit("Use either --version or --min-version/--max-version")
340
+
else:
341
+
args.min_version = args.version
342
+
args.max_version = args.version
143
343
144
-
if not args.min_version:
145
-
args.min_version = sphinx_versions[0]
344
+
sphinx_versions = sorted(list(SPHINX_REQUIREMENTS.keys()))
146
345
147
-
if not args.max_version:
148
-
args.max_version = sphinx_versions[-1]
346
+
if not args.min_version:
347
+
args.min_version = sphinx_versions[0]
149
348
150
-
first_run = True
151
-
cur_requirements = {}
152
-
built_time = {}
349
+
if not args.max_version:
350
+
args.max_version = sphinx_versions[-1]
153
351
154
-
for cur_ver, new_reqs in sphinx_requirements.items():
155
-
cur_requirements.update(new_reqs)
352
+
venv = SphinxVenv()
353
+
await venv.run(args)
156
354
157
-
if cur_ver in python_changes:
158
-
python_bin = python_changes[cur_ver]
159
355
160
-
ver = ".".join(map(str, cur_ver))
161
-
162
-
if args.min_version:
163
-
if cur_ver < args.min_version:
164
-
continue
165
-
166
-
if args.max_version:
167
-
if cur_ver > args.max_version:
168
-
break
169
-
170
-
if not first_run and args.wait_input and args.make:
171
-
ret = input("Press Enter to continue or 'a' to abort: ").strip().lower()
172
-
if ret == "a":
173
-
print("Aborted.")
174
-
sys.exit()
175
-
else:
176
-
first_run = False
177
-
178
-
venv_dir = f"Sphinx_{ver}"
179
-
req_file = f"requirements_{ver}.txt"
180
-
181
-
print(f"\nSphinx {ver} with {python_bin}")
182
-
183
-
# Create venv
184
-
run([python_bin, "-m", "venv", venv_dir], check=True)
185
-
pip = os.path.join(venv_dir, "bin/pip")
186
-
187
-
# Create install list
188
-
reqs = []
189
-
for pkg, verstr in cur_requirements.items():
190
-
reqs.append(f"{pkg}=={verstr}")
191
-
192
-
reqs.append(f"Sphinx=={ver}")
193
-
194
-
run([pip, "install"] + reqs, check=True)
195
-
196
-
# Freeze environment
197
-
result = run([pip, "freeze"], capture_output=True, text=True, check=True)
198
-
199
-
# Pip install succeeded. Write requirements file
200
-
if args.write:
201
-
with open(req_file, "w", encoding="utf-8") as fp:
202
-
fp.write(result.stdout)
203
-
204
-
if args.make:
205
-
start_time = time.time()
206
-
207
-
# Prepare a venv environment
208
-
env = os.environ.copy()
209
-
bin_dir = os.path.join(venv_dir, "bin")
210
-
env["PATH"] = bin_dir + ":" + env["PATH"]
211
-
env["VIRTUAL_ENV"] = venv_dir
212
-
if "PYTHONHOME" in env:
213
-
del env["PYTHONHOME"]
214
-
215
-
# Test doc build
216
-
run(["make", "cleandocs"], env=env, check=True)
217
-
make = ["make"] + args.make_args + ["htmldocs"]
218
-
219
-
print(f". {bin_dir}/activate")
220
-
print(" ".join(make))
221
-
print("deactivate")
222
-
run(make, env=env, check=True)
223
-
224
-
end_time = time.time()
225
-
elapsed_time = end_time - start_time
226
-
hours, minutes = divmod(elapsed_time, 3600)
227
-
minutes, seconds = divmod(minutes, 60)
228
-
229
-
hours = int(hours)
230
-
minutes = int(minutes)
231
-
seconds = int(seconds)
232
-
233
-
built_time[ver] = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
234
-
235
-
print(f"Finished doc build for Sphinx {ver}. Elapsed time: {built_time[ver]}")
236
-
237
-
if args.make:
238
-
print()
239
-
print("Summary:")
240
-
for ver, elapsed_time in sorted(built_time.items()):
241
-
print(f"\tSphinx {ver} elapsed time: {elapsed_time}")
356
+
# Call main method
357
+
if __name__ == "__main__":
358
+
asyncio.run(main())