Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'core-entry-2024-01-08' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull generic syscall updates from Ingo Molnar:
"Move various entry functions from kernel/entry/common.c to a header
file, and always-inline them, to improve syscall entry performance
on s390 by ~11%"
* tag 'core-entry-2024-01-08' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
entry: Move syscall_enter_from_user_mode() to header file
entry: Move enter_from_user_mode() to header file
entry: Move exit to usermode functions to header file
+103
-100
+91
-4
include/linux/entry-common.h
+91
-4
include/linux/entry-common.h
···
7
7
#include <linux/syscalls.h>
8
8
#include <linux/seccomp.h>
9
9
#include <linux/sched.h>
10
+
#include <linux/context_tracking.h>
11
+
#include <linux/livepatch.h>
12
+
#include <linux/resume_user_mode.h>
13
+
#include <linux/tick.h>
14
+
#include <linux/kmsan.h>
10
15
11
16
#include <asm/entry-common.h>
12
17
···
103
98
* done between establishing state and enabling interrupts. The caller must
104
99
* enable interrupts before invoking syscall_enter_from_user_mode_work().
105
100
*/
106
-
void enter_from_user_mode(struct pt_regs *regs);
101
+
static __always_inline void enter_from_user_mode(struct pt_regs *regs)
102
+
{
103
+
arch_enter_from_user_mode(regs);
104
+
lockdep_hardirqs_off(CALLER_ADDR0);
105
+
106
+
CT_WARN_ON(__ct_state() != CONTEXT_USER);
107
+
user_exit_irqoff();
108
+
109
+
instrumentation_begin();
110
+
kmsan_unpoison_entry_regs(regs);
111
+
trace_hardirqs_off_finish();
112
+
instrumentation_end();
113
+
}
107
114
108
115
/**
109
116
* syscall_enter_from_user_mode_prepare - Establish state and enable interrupts
···
133
116
* to be done between establishing state and handling user mode entry work.
134
117
*/
135
118
void syscall_enter_from_user_mode_prepare(struct pt_regs *regs);
119
+
120
+
long syscall_trace_enter(struct pt_regs *regs, long syscall,
121
+
unsigned long work);
136
122
137
123
/**
138
124
* syscall_enter_from_user_mode_work - Check and handle work before invoking
···
160
140
* ptrace_report_syscall_entry(), __secure_computing(), trace_sys_enter()
161
141
* 2) Invocation of audit_syscall_entry()
162
142
*/
163
-
long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall);
143
+
static __always_inline long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall)
144
+
{
145
+
unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
146
+
147
+
if (work & SYSCALL_WORK_ENTER)
148
+
syscall = syscall_trace_enter(regs, syscall, work);
149
+
150
+
return syscall;
151
+
}
164
152
165
153
/**
166
154
* syscall_enter_from_user_mode - Establish state and check and handle work
···
187
159
* Returns: The original or a modified syscall number. See
188
160
* syscall_enter_from_user_mode_work() for further explanation.
189
161
*/
190
-
long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall);
162
+
static __always_inline long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)
163
+
{
164
+
long ret;
165
+
166
+
enter_from_user_mode(regs);
167
+
168
+
instrumentation_begin();
169
+
local_irq_enable();
170
+
ret = syscall_enter_from_user_mode_work(regs, syscall);
171
+
instrumentation_end();
172
+
173
+
return ret;
174
+
}
191
175
192
176
/**
193
177
* local_irq_enable_exit_to_user - Exit to user variant of local_irq_enable()
···
299
259
void arch_do_signal_or_restart(struct pt_regs *regs);
300
260
301
261
/**
262
+
* exit_to_user_mode_loop - do any pending work before leaving to user space
263
+
*/
264
+
unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
265
+
unsigned long ti_work);
266
+
267
+
/**
268
+
* exit_to_user_mode_prepare - call exit_to_user_mode_loop() if required
269
+
* @regs: Pointer to pt_regs on entry stack
270
+
*
271
+
* 1) check that interrupts are disabled
272
+
* 2) call tick_nohz_user_enter_prepare()
273
+
* 3) call exit_to_user_mode_loop() if any flags from
274
+
* EXIT_TO_USER_MODE_WORK are set
275
+
* 4) check that interrupts are still disabled
276
+
*/
277
+
static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs)
278
+
{
279
+
unsigned long ti_work;
280
+
281
+
lockdep_assert_irqs_disabled();
282
+
283
+
/* Flush pending rcuog wakeup before the last need_resched() check */
284
+
tick_nohz_user_enter_prepare();
285
+
286
+
ti_work = read_thread_flags();
287
+
if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
288
+
ti_work = exit_to_user_mode_loop(regs, ti_work);
289
+
290
+
arch_exit_to_user_mode_prepare(regs, ti_work);
291
+
292
+
/* Ensure that kernel state is sane for a return to userspace */
293
+
kmap_assert_nomap();
294
+
lockdep_assert_irqs_disabled();
295
+
lockdep_sys_exit();
296
+
}
297
+
298
+
/**
302
299
* exit_to_user_mode - Fixup state when exiting to user mode
303
300
*
304
301
* Syscall/interrupt exit enables interrupts, but the kernel state is
···
353
276
* non-instrumentable.
354
277
* The caller has to invoke syscall_exit_to_user_mode_work() before this.
355
278
*/
356
-
void exit_to_user_mode(void);
279
+
static __always_inline void exit_to_user_mode(void)
280
+
{
281
+
instrumentation_begin();
282
+
trace_hardirqs_on_prepare();
283
+
lockdep_hardirqs_on_prepare();
284
+
instrumentation_end();
285
+
286
+
user_enter_irqoff();
287
+
arch_exit_to_user_mode();
288
+
lockdep_hardirqs_on(CALLER_ADDR0);
289
+
}
357
290
358
291
/**
359
292
* syscall_exit_to_user_mode_work - Handle work before returning to user mode
+12
-96
kernel/entry/common.c
+12
-96
kernel/entry/common.c
···
15
15
#define CREATE_TRACE_POINTS
16
16
#include <trace/events/syscalls.h>
17
17
18
-
/* See comment for enter_from_user_mode() in entry-common.h */
19
-
static __always_inline void __enter_from_user_mode(struct pt_regs *regs)
20
-
{
21
-
arch_enter_from_user_mode(regs);
22
-
lockdep_hardirqs_off(CALLER_ADDR0);
23
-
24
-
CT_WARN_ON(__ct_state() != CONTEXT_USER);
25
-
user_exit_irqoff();
26
-
27
-
instrumentation_begin();
28
-
kmsan_unpoison_entry_regs(regs);
29
-
trace_hardirqs_off_finish();
30
-
instrumentation_end();
31
-
}
32
-
33
-
void noinstr enter_from_user_mode(struct pt_regs *regs)
34
-
{
35
-
__enter_from_user_mode(regs);
36
-
}
37
-
38
18
static inline void syscall_enter_audit(struct pt_regs *regs, long syscall)
39
19
{
40
20
if (unlikely(audit_context())) {
···
25
45
}
26
46
}
27
47
28
-
static long syscall_trace_enter(struct pt_regs *regs, long syscall,
48
+
long syscall_trace_enter(struct pt_regs *regs, long syscall,
29
49
unsigned long work)
30
50
{
31
51
long ret = 0;
···
65
85
return ret ? : syscall;
66
86
}
67
87
68
-
static __always_inline long
69
-
__syscall_enter_from_user_work(struct pt_regs *regs, long syscall)
70
-
{
71
-
unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
72
-
73
-
if (work & SYSCALL_WORK_ENTER)
74
-
syscall = syscall_trace_enter(regs, syscall, work);
75
-
76
-
return syscall;
77
-
}
78
-
79
-
long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall)
80
-
{
81
-
return __syscall_enter_from_user_work(regs, syscall);
82
-
}
83
-
84
-
noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)
85
-
{
86
-
long ret;
87
-
88
-
__enter_from_user_mode(regs);
89
-
90
-
instrumentation_begin();
91
-
local_irq_enable();
92
-
ret = __syscall_enter_from_user_work(regs, syscall);
93
-
instrumentation_end();
94
-
95
-
return ret;
96
-
}
97
-
98
88
noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs)
99
89
{
100
-
__enter_from_user_mode(regs);
90
+
enter_from_user_mode(regs);
101
91
instrumentation_begin();
102
92
local_irq_enable();
103
93
instrumentation_end();
104
-
}
105
-
106
-
/* See comment for exit_to_user_mode() in entry-common.h */
107
-
static __always_inline void __exit_to_user_mode(void)
108
-
{
109
-
instrumentation_begin();
110
-
trace_hardirqs_on_prepare();
111
-
lockdep_hardirqs_on_prepare();
112
-
instrumentation_end();
113
-
114
-
user_enter_irqoff();
115
-
arch_exit_to_user_mode();
116
-
lockdep_hardirqs_on(CALLER_ADDR0);
117
-
}
118
-
119
-
void noinstr exit_to_user_mode(void)
120
-
{
121
-
__exit_to_user_mode();
122
94
}
123
95
124
96
/* Workaround to allow gradual conversion of architecture code */
125
97
void __weak arch_do_signal_or_restart(struct pt_regs *regs) { }
126
98
127
-
static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
128
-
unsigned long ti_work)
99
+
/**
100
+
* exit_to_user_mode_loop - do any pending work before leaving to user space
101
+
* @regs: Pointer to pt_regs on entry stack
102
+
* @ti_work: TIF work flags as read by the caller
103
+
*/
104
+
__always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
105
+
unsigned long ti_work)
129
106
{
130
107
/*
131
108
* Before returning to user space ensure that all pending work
···
125
188
126
189
/* Return the latest work state for arch_exit_to_user_mode() */
127
190
return ti_work;
128
-
}
129
-
130
-
static void exit_to_user_mode_prepare(struct pt_regs *regs)
131
-
{
132
-
unsigned long ti_work;
133
-
134
-
lockdep_assert_irqs_disabled();
135
-
136
-
/* Flush pending rcuog wakeup before the last need_resched() check */
137
-
tick_nohz_user_enter_prepare();
138
-
139
-
ti_work = read_thread_flags();
140
-
if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
141
-
ti_work = exit_to_user_mode_loop(regs, ti_work);
142
-
143
-
arch_exit_to_user_mode_prepare(regs, ti_work);
144
-
145
-
/* Ensure that kernel state is sane for a return to userspace */
146
-
kmap_assert_nomap();
147
-
lockdep_assert_irqs_disabled();
148
-
lockdep_sys_exit();
149
191
}
150
192
151
193
/*
···
211
295
instrumentation_begin();
212
296
__syscall_exit_to_user_mode_work(regs);
213
297
instrumentation_end();
214
-
__exit_to_user_mode();
298
+
exit_to_user_mode();
215
299
}
216
300
217
301
noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs)
218
302
{
219
-
__enter_from_user_mode(regs);
303
+
enter_from_user_mode(regs);
220
304
}
221
305
222
306
noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs)
···
224
308
instrumentation_begin();
225
309
exit_to_user_mode_prepare(regs);
226
310
instrumentation_end();
227
-
__exit_to_user_mode();
311
+
exit_to_user_mode();
228
312
}
229
313
230
314
noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)