Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
jobserver: Split up the big try: block
The parsing of jobserver options is done in a massive try: block that hides
problems and (perhaps) bugs. Split up that block and make the logic
explicit by moving the initial parsing of MAKEFLAGS out of that block. Add
warnings in the places things can go wrong.
Reviewed-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
Signed-off-by: Jonathan Corbet <corbet@lwn.net>
+90
-53
+90
-53
tools/lib/python/jobserver.py
+90
-53
tools/lib/python/jobserver.py
···
35
35
import subprocess
36
36
import sys
37
37
38
+
def warn(text, *args):
39
+
print(f'WARNING: {text}', *args, file = sys.stderr)
40
+
38
41
class JobserverExec:
39
42
"""
40
43
Claim all slots from make using POSIX Jobserver.
···
61
58
62
59
if self.is_open:
63
60
return
64
-
65
-
try:
66
-
# Fetch the make environment options.
67
-
flags = os.environ["MAKEFLAGS"]
68
-
# Look for "--jobserver=R,W"
69
-
# Note that GNU Make has used --jobserver-fds and --jobserver-auth
70
-
# so this handles all of them.
71
-
opts = [x for x in flags.split(" ") if x.startswith("--jobserver")]
72
-
73
-
# Parse out R,W file descriptor numbers and set them nonblocking.
74
-
# If the MAKEFLAGS variable contains multiple instances of the
75
-
# --jobserver-auth= option, the last one is relevant.
76
-
fds = opts[-1].split("=", 1)[1]
77
-
78
-
# Starting with GNU Make 4.4, named pipes are used for reader
79
-
# and writer.
80
-
# Example argument: --jobserver-auth=fifo:/tmp/GMfifo8134
81
-
_, _, path = fds.partition("fifo:")
82
-
83
-
if path:
61
+
self.is_open = True # We only try once
62
+
self.claim = None
63
+
#
64
+
# Check the make flags for "--jobserver=R,W"
65
+
# Note that GNU Make has used --jobserver-fds and --jobserver-auth
66
+
# so this handles all of them.
67
+
#
68
+
flags = os.environ.get('MAKEFLAGS', '')
69
+
opts = [x for x in flags.split(" ") if x.startswith("--jobserver")]
70
+
if not opts:
71
+
return
72
+
#
73
+
# Separate out the provided file descriptors
74
+
#
75
+
split_opt = opts[-1].split('=', 1)
76
+
if len(split_opt) != 2:
77
+
warn('unparseable option:', opts[-1])
78
+
return
79
+
fds = split_opt[1]
80
+
#
81
+
# As of GNU Make 4.4, we'll be looking for a named pipe
82
+
# identified as fifo:path
83
+
#
84
+
if fds.startswith('fifo:'):
85
+
path = fds[len('fifo:'):]
86
+
try:
84
87
self.reader = os.open(path, os.O_RDONLY | os.O_NONBLOCK)
85
88
self.writer = os.open(path, os.O_WRONLY)
86
-
else:
87
-
self.reader, self.writer = [int(x) for x in fds.split(",", 1)]
89
+
except (OSError, IOError):
90
+
warn('unable to open jobserver pipe', path)
91
+
return
92
+
#
93
+
# Otherwise look for integer file-descriptor numbers.
94
+
#
95
+
else:
96
+
split_fds = fds.split(',')
97
+
if len(split_fds) != 2:
98
+
warn('malformed jobserver file descriptors:', fds)
99
+
return
100
+
try:
101
+
self.reader = int(split_fds[0])
102
+
self.writer = int(split_fds[1])
103
+
except ValueError:
104
+
warn('non-integer jobserver file-descriptors:', fds)
105
+
return
106
+
try:
107
+
#
88
108
# Open a private copy of reader to avoid setting nonblocking
89
109
# on an unexpecting process with the same reader fd.
90
-
self.reader = os.open("/proc/self/fd/%d" % (self.reader),
110
+
#
111
+
self.reader = os.open(f"/proc/self/fd/{self.reader}",
91
112
os.O_RDONLY | os.O_NONBLOCK)
92
-
93
-
# Read out as many jobserver slots as possible
94
-
while True:
95
-
try:
96
-
slot = os.read(self.reader, 8)
97
-
if not slot:
98
-
# Clear self.jobs to prevent us from probably writing incorrect file.
99
-
self.jobs = b""
100
-
raise ValueError("unexpected empty token from jobserver fd, invalid '--jobserver-auth=' setting?")
101
-
self.jobs += slot
102
-
except (OSError, IOError) as e:
103
-
if e.errno == errno.EWOULDBLOCK:
104
-
# Stop at the end of the jobserver queue.
105
-
break
106
-
# If something went wrong, give back the jobs.
107
-
if self.jobs:
108
-
os.write(self.writer, self.jobs)
109
-
raise e
110
-
111
-
# Add a bump for our caller's reserveration, since we're just going
112
-
# to sit here blocked on our child.
113
-
self.claim = len(self.jobs) + 1
114
-
115
-
except (KeyError, IndexError, ValueError, OSError, IOError) as e:
116
-
print(f"jobserver: warning: {repr(e)}", file=sys.stderr)
117
-
# Any missing environment strings or bad fds should result in just
118
-
# not being parallel.
119
-
self.claim = None
120
-
121
-
self.is_open = True
113
+
except (IOError, OSError) as e:
114
+
warn('Unable to reopen jobserver read-side pipe:', repr(e))
115
+
return
116
+
#
117
+
# OK, we have the channel to the job server; read out as many jobserver
118
+
# slots as possible.
119
+
#
120
+
while True:
121
+
try:
122
+
slot = os.read(self.reader, 8)
123
+
if not slot:
124
+
#
125
+
# Something went wrong. Clear self.jobs to avoid writing
126
+
# weirdness back to the jobserver and give up.
127
+
self.jobs = b""
128
+
warn("unexpected empty token from jobserver;"
129
+
" possible invalid '--jobserver-auth=' setting")
130
+
self.claim = None
131
+
return
132
+
except (OSError, IOError) as e:
133
+
#
134
+
# If there is nothing more to read then we are done.
135
+
#
136
+
if e.errno == errno.EWOULDBLOCK:
137
+
break
138
+
#
139
+
# Anything else says that something went weird; give back
140
+
# the jobs and give up.
141
+
#
142
+
if self.jobs:
143
+
os.write(self.writer, self.jobs)
144
+
self.claim = None
145
+
warn('error reading from jobserver pipe', repr(e))
146
+
return
147
+
self.jobs += slot
148
+
#
149
+
# Add a bump for our caller's reserveration, since we're just going
150
+
# to sit here blocked on our child.
151
+
#
152
+
self.claim = len(self.jobs) + 1
122
153
123
154
def close(self):
124
155
"""Return all reserved slots to Jobserver"""