tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

rseq: Introduce struct rseq_data

In preparation for a major rewrite of this code, provide a data structure
for rseq management.

Put all the rseq related data into it (except for the debug part), which
allows to simplify fork/execve by using memset() and memcpy() instead of
adding new fields to initialize over and over.

Create a storage struct for event management as well and put the
sched_switch event and a indicator for RSEQ on a task into it as a
start. That uses a union, which allows to mask and clear the whole lot
efficiently.

The indicators are explicitly not a bit field. Bit fields generate abysmal
code.

The boolean members are defined as u8 as that actually guarantees that it
fits. There seem to be strange architecture ABIs which need more than 8
bits for a boolean.

The has_rseq member is redundant vs. task::rseq, but it turns out that
boolean operations and quick checks on the union generate better code than
fiddling with separate entities and data types.

This struct will be extended over time to carry more information.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251027084306.527086690@linutronix.de

authored by

Thomas Gleixner and committed by
Ingo Molnar faba9d25 566d8015

+110 -72
+22 -26
include/linux/rseq.h
··· 9 9 10 10 static inline void rseq_handle_notify_resume(struct pt_regs *regs) 11 11 { 12 - if (current->rseq) 12 + if (current->rseq.event.has_rseq) 13 13 __rseq_handle_notify_resume(NULL, regs); 14 14 } 15 15 16 16 static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) 17 17 { 18 - if (current->rseq) { 19 - current->rseq_event_pending = true; 18 + if (current->rseq.event.has_rseq) { 19 + current->rseq.event.sched_switch = true; 20 20 __rseq_handle_notify_resume(ksig, regs); 21 21 } 22 22 } 23 23 24 24 static inline void rseq_sched_switch_event(struct task_struct *t) 25 25 { 26 - if (t->rseq) { 27 - t->rseq_event_pending = true; 26 + if (t->rseq.event.has_rseq) { 27 + t->rseq.event.sched_switch = true; 28 28 set_tsk_thread_flag(t, TIF_NOTIFY_RESUME); 29 29 } 30 30 } ··· 32 32 static __always_inline void rseq_exit_to_user_mode(void) 33 33 { 34 34 if (IS_ENABLED(CONFIG_DEBUG_RSEQ)) { 35 - if (WARN_ON_ONCE(current->rseq && current->rseq_event_pending)) 36 - current->rseq_event_pending = false; 35 + if (WARN_ON_ONCE(current->rseq.event.has_rseq && 36 + current->rseq.event.events)) 37 + current->rseq.event.events = 0; 37 38 } 38 39 } 39 40 ··· 50 49 */ 51 50 static inline void rseq_virt_userspace_exit(void) 52 51 { 53 - if (current->rseq_event_pending) 52 + if (current->rseq.event.sched_switch) 54 53 set_tsk_thread_flag(current, TIF_NOTIFY_RESUME); 54 + } 55 + 56 + static inline void rseq_reset(struct task_struct *t) 57 + { 58 + memset(&t->rseq, 0, sizeof(t->rseq)); 59 + } 60 + 61 + static inline void rseq_execve(struct task_struct *t) 62 + { 63 + rseq_reset(t); 55 64 } 56 65 57 66 /* ··· 70 59 */ 71 60 static inline void rseq_fork(struct task_struct *t, u64 clone_flags) 72 61 { 73 - if (clone_flags & CLONE_VM) { 74 - t->rseq = NULL; 75 - t->rseq_len = 0; 76 - t->rseq_sig = 0; 77 - t->rseq_event_pending = false; 78 - } else { 62 + if (clone_flags & CLONE_VM) 63 + rseq_reset(t); 64 + else 79 65 t->rseq = current->rseq; 80 - t->rseq_len = current->rseq_len; 81 - t->rseq_sig = current->rseq_sig; 82 - t->rseq_event_pending = current->rseq_event_pending; 83 - } 84 - } 85 - 86 - static inline void rseq_execve(struct task_struct *t) 87 - { 88 - t->rseq = NULL; 89 - t->rseq_len = 0; 90 - t->rseq_sig = 0; 91 - t->rseq_event_pending = false; 92 66 } 93 67 94 68 #else /* CONFIG_RSEQ */
+51
include/linux/rseq_types.h
··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 + #ifndef _LINUX_RSEQ_TYPES_H 3 + #define _LINUX_RSEQ_TYPES_H 4 + 5 + #include <linux/types.h> 6 + 7 + #ifdef CONFIG_RSEQ 8 + struct rseq; 9 + 10 + /** 11 + * struct rseq_event - Storage for rseq related event management 12 + * @all: Compound to initialize and clear the data efficiently 13 + * @events: Compound to access events with a single load/store 14 + * @sched_switch: True if the task was scheduled out 15 + * @has_rseq: True if the task has a rseq pointer installed 16 + */ 17 + struct rseq_event { 18 + union { 19 + u32 all; 20 + struct { 21 + union { 22 + u16 events; 23 + struct { 24 + u8 sched_switch; 25 + }; 26 + }; 27 + 28 + u8 has_rseq; 29 + }; 30 + }; 31 + }; 32 + 33 + /** 34 + * struct rseq_data - Storage for all rseq related data 35 + * @usrptr: Pointer to the registered user space RSEQ memory 36 + * @len: Length of the RSEQ region 37 + * @sig: Signature of critial section abort IPs 38 + * @event: Storage for event management 39 + */ 40 + struct rseq_data { 41 + struct rseq __user *usrptr; 42 + u32 len; 43 + u32 sig; 44 + struct rseq_event event; 45 + }; 46 + 47 + #else /* CONFIG_RSEQ */ 48 + struct rseq_data { }; 49 + #endif /* !CONFIG_RSEQ */ 50 + 51 + #endif
+3 -11
include/linux/sched.h
··· 41 41 #include <linux/task_io_accounting.h> 42 42 #include <linux/posix-timers_types.h> 43 43 #include <linux/restart_block.h> 44 + #include <linux/rseq_types.h> 44 45 #include <uapi/linux/rseq.h> 45 46 #include <linux/seqlock_types.h> 46 47 #include <linux/kcsan.h> ··· 1407 1406 unsigned long numa_pages_migrated; 1408 1407 #endif /* CONFIG_NUMA_BALANCING */ 1409 1408 1410 - #ifdef CONFIG_RSEQ 1411 - struct rseq __user *rseq; 1412 - u32 rseq_len; 1413 - u32 rseq_sig; 1414 - /* 1415 - * RmW on rseq_event_pending must be performed atomically 1416 - * with respect to preemption. 1417 - */ 1418 - bool rseq_event_pending; 1419 - # ifdef CONFIG_DEBUG_RSEQ 1409 + struct rseq_data rseq; 1410 + #ifdef CONFIG_DEBUG_RSEQ 1420 1411 /* 1421 1412 * This is a place holder to save a copy of the rseq fields for 1422 1413 * validation of read-only fields. The struct rseq has a ··· 1416 1423 * directly. Reserve a size large enough for the known fields. 1417 1424 */ 1418 1425 char rseq_fields[sizeof(struct rseq)]; 1419 - # endif 1420 1426 #endif 1421 1427 1422 1428 #ifdef CONFIG_SCHED_MM_CID
+3 -3
kernel/ptrace.c
··· 793 793 unsigned long size, void __user *data) 794 794 { 795 795 struct ptrace_rseq_configuration conf = { 796 - .rseq_abi_pointer = (u64)(uintptr_t)task->rseq, 797 - .rseq_abi_size = task->rseq_len, 798 - .signature = task->rseq_sig, 796 + .rseq_abi_pointer = (u64)(uintptr_t)task->rseq.usrptr, 797 + .rseq_abi_size = task->rseq.len, 798 + .signature = task->rseq.sig, 799 799 .flags = 0, 800 800 }; 801 801
+31 -32
kernel/rseq.c
··· 103 103 DEFAULT_RATELIMIT_INTERVAL, 104 104 DEFAULT_RATELIMIT_BURST); 105 105 u32 cpu_id_start, cpu_id, node_id, mm_cid; 106 - struct rseq __user *rseq = t->rseq; 106 + struct rseq __user *rseq = t->rseq.usrptr; 107 107 108 108 /* 109 109 * Validate fields which are required to be read-only by 110 110 * user-space. 111 111 */ 112 - if (!user_read_access_begin(rseq, t->rseq_len)) 112 + if (!user_read_access_begin(rseq, t->rseq.len)) 113 113 goto efault; 114 114 unsafe_get_user(cpu_id_start, &rseq->cpu_id_start, efault_end); 115 115 unsafe_get_user(cpu_id, &rseq->cpu_id, efault_end); ··· 147 147 * Update an rseq field and its in-kernel copy in lock-step to keep a coherent 148 148 * state. 149 149 */ 150 - #define rseq_unsafe_put_user(t, value, field, error_label) \ 151 - do { \ 152 - unsafe_put_user(value, &t->rseq->field, error_label); \ 153 - rseq_kernel_fields(t)->field = value; \ 150 + #define rseq_unsafe_put_user(t, value, field, error_label) \ 151 + do { \ 152 + unsafe_put_user(value, &t->rseq.usrptr->field, error_label); \ 153 + rseq_kernel_fields(t)->field = value; \ 154 154 } while (0) 155 155 156 156 #else ··· 160 160 } 161 161 162 162 #define rseq_unsafe_put_user(t, value, field, error_label) \ 163 - unsafe_put_user(value, &t->rseq->field, error_label) 163 + unsafe_put_user(value, &t->rseq.usrptr->field, error_label) 164 164 #endif 165 165 166 166 static int rseq_update_cpu_node_id(struct task_struct *t) 167 167 { 168 - struct rseq __user *rseq = t->rseq; 168 + struct rseq __user *rseq = t->rseq.usrptr; 169 169 u32 cpu_id = raw_smp_processor_id(); 170 170 u32 node_id = cpu_to_node(cpu_id); 171 171 u32 mm_cid = task_mm_cid(t); ··· 176 176 if (rseq_validate_ro_fields(t)) 177 177 goto efault; 178 178 WARN_ON_ONCE((int) mm_cid < 0); 179 - if (!user_write_access_begin(rseq, t->rseq_len)) 179 + if (!user_write_access_begin(rseq, t->rseq.len)) 180 180 goto efault; 181 181 182 182 rseq_unsafe_put_user(t, cpu_id, cpu_id_start, efault_end); ··· 201 201 202 202 static int rseq_reset_rseq_cpu_node_id(struct task_struct *t) 203 203 { 204 - struct rseq __user *rseq = t->rseq; 204 + struct rseq __user *rseq = t->rseq.usrptr; 205 205 u32 cpu_id_start = 0, cpu_id = RSEQ_CPU_ID_UNINITIALIZED, node_id = 0, 206 206 mm_cid = 0; 207 207 ··· 211 211 if (rseq_validate_ro_fields(t)) 212 212 goto efault; 213 213 214 - if (!user_write_access_begin(rseq, t->rseq_len)) 214 + if (!user_write_access_begin(rseq, t->rseq.len)) 215 215 goto efault; 216 216 217 217 /* ··· 272 272 u32 sig; 273 273 int ret; 274 274 275 - ret = rseq_get_rseq_cs_ptr_val(t->rseq, &ptr); 275 + ret = rseq_get_rseq_cs_ptr_val(t->rseq.usrptr, &ptr); 276 276 if (ret) 277 277 return ret; 278 278 ··· 305 305 if (ret) 306 306 return ret; 307 307 308 - if (current->rseq_sig != sig) { 308 + if (current->rseq.sig != sig) { 309 309 printk_ratelimited(KERN_WARNING 310 310 "Possible attack attempt. Unexpected rseq signature 0x%x, expecting 0x%x (pid=%d, addr=%p).\n", 311 - sig, current->rseq_sig, current->pid, usig); 311 + sig, current->rseq.sig, current->pid, usig); 312 312 return -EINVAL; 313 313 } 314 314 return 0; ··· 338 338 return -EINVAL; 339 339 340 340 /* Get thread flags. */ 341 - ret = get_user(flags, &t->rseq->flags); 341 + ret = get_user(flags, &t->rseq.usrptr->flags); 342 342 if (ret) 343 343 return ret; 344 344 ··· 392 392 * Clear the rseq_cs pointer and return. 393 393 */ 394 394 if (!in_rseq_cs(ip, &rseq_cs)) 395 - return clear_rseq_cs(t->rseq); 395 + return clear_rseq_cs(t->rseq.usrptr); 396 396 ret = rseq_check_flags(t, rseq_cs.flags); 397 397 if (ret < 0) 398 398 return ret; 399 399 if (!abort) 400 400 return 0; 401 - ret = clear_rseq_cs(t->rseq); 401 + ret = clear_rseq_cs(t->rseq.usrptr); 402 402 if (ret) 403 403 return ret; 404 404 trace_rseq_ip_fixup(ip, rseq_cs.start_ip, rseq_cs.post_commit_offset, ··· 460 460 * inconsistencies. 461 461 */ 462 462 scoped_guard(RSEQ_EVENT_GUARD) { 463 - event = t->rseq_event_pending; 464 - t->rseq_event_pending = false; 463 + event = t->rseq.event.sched_switch; 464 + t->rseq.event.sched_switch = false; 465 465 } 466 466 467 467 if (!IS_ENABLED(CONFIG_DEBUG_RSEQ) && !event) ··· 492 492 struct task_struct *t = current; 493 493 struct rseq_cs rseq_cs; 494 494 495 - if (!t->rseq) 495 + if (!t->rseq.usrptr) 496 496 return; 497 497 if (rseq_get_rseq_cs(t, &rseq_cs) || in_rseq_cs(ip, &rseq_cs)) 498 498 force_sig(SIGSEGV); ··· 511 511 if (flags & ~RSEQ_FLAG_UNREGISTER) 512 512 return -EINVAL; 513 513 /* Unregister rseq for current thread. */ 514 - if (current->rseq != rseq || !current->rseq) 514 + if (current->rseq.usrptr != rseq || !current->rseq.usrptr) 515 515 return -EINVAL; 516 - if (rseq_len != current->rseq_len) 516 + if (rseq_len != current->rseq.len) 517 517 return -EINVAL; 518 - if (current->rseq_sig != sig) 518 + if (current->rseq.sig != sig) 519 519 return -EPERM; 520 520 ret = rseq_reset_rseq_cpu_node_id(current); 521 521 if (ret) 522 522 return ret; 523 - current->rseq = NULL; 524 - current->rseq_sig = 0; 525 - current->rseq_len = 0; 523 + rseq_reset(current); 526 524 return 0; 527 525 } 528 526 529 527 if (unlikely(flags)) 530 528 return -EINVAL; 531 529 532 - if (current->rseq) { 530 + if (current->rseq.usrptr) { 533 531 /* 534 532 * If rseq is already registered, check whether 535 533 * the provided address differs from the prior 536 534 * one. 537 535 */ 538 - if (current->rseq != rseq || rseq_len != current->rseq_len) 536 + if (current->rseq.usrptr != rseq || rseq_len != current->rseq.len) 539 537 return -EINVAL; 540 - if (current->rseq_sig != sig) 538 + if (current->rseq.sig != sig) 541 539 return -EPERM; 542 540 /* Already registered. */ 543 541 return -EBUSY; ··· 584 586 * Activate the registration by setting the rseq area address, length 585 587 * and signature in the task struct. 586 588 */ 587 - current->rseq = rseq; 588 - current->rseq_len = rseq_len; 589 - current->rseq_sig = sig; 589 + current->rseq.usrptr = rseq; 590 + current->rseq.len = rseq_len; 591 + current->rseq.sig = sig; 590 592 591 593 /* 592 594 * If rseq was previously inactive, and has just been 593 595 * registered, ensure the cpu_id_start and cpu_id fields 594 596 * are updated before returning to user-space. 595 597 */ 598 + current->rseq.event.has_rseq = true; 596 599 rseq_sched_switch_event(current); 597 600 598 601 return 0;