exit: combine work under lock in synchronize_group_exit() and coredump_task_exit()

This reduces single-threaded overhead as it avoids one lock+irq trip on
exit.

It also improves scalability of spawning and killing threads within one
process (just shy of 5% when doing it on 24 cores on my test jig).

Both routines are moved below kcov and kmsan exit, which should be
harmless.

Link: https://lkml.kernel.org/r/20250319195436.1864415-1-mjguzik@gmail.com
Signed-off-by: Mateusz Guzik <mjguzik@gmail.com>
Reviewed-by: Oleg Nesterov <oleg@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Mateusz Guzik and committed by

Andrew Morton 1 year ago fe6f600c 3eff6a3e

+32 -36

1 changed file

expand all

kernel

exit.c

+32 -36

kernel/exit.c

··· 415 415 } 416 416 } 417 417 418 - static void coredump_task_exit(struct task_struct *tsk) 418 + static void coredump_task_exit(struct task_struct *tsk, 419 + struct core_state *core_state) 419 420 { 420 - struct core_state *core_state; 421 + struct core_thread self; 421 422 423 + self.task = tsk; 424 + if (self.task->flags & PF_SIGNALED) 425 + self.next = xchg(&core_state->dumper.next, &self); 426 + else 427 + self.task = NULL; 422 428 /* 423 - * Serialize with any possible pending coredump. 424 - * We must hold siglock around checking core_state 425 - * and setting PF_POSTCOREDUMP. The core-inducing thread 426 - * will increment ->nr_threads for each thread in the 427 - * group without PF_POSTCOREDUMP set. 429 + * Implies mb(), the result of xchg() must be visible 430 + * to core_state->dumper. 428 431 */ 429 - spin_lock_irq(&tsk->sighand->siglock); 430 - tsk->flags |= PF_POSTCOREDUMP; 431 - core_state = tsk->signal->core_state; 432 - spin_unlock_irq(&tsk->sighand->siglock); 433 - if (core_state) { 434 - struct core_thread self; 432 + if (atomic_dec_and_test(&core_state->nr_threads)) 433 + complete(&core_state->startup); 435 434 436 - self.task = current; 437 - if (self.task->flags & PF_SIGNALED) 438 - self.next = xchg(&core_state->dumper.next, &self); 439 - else 440 - self.task = NULL; 441 - /* 442 - * Implies mb(), the result of xchg() must be visible 443 - * to core_state->dumper. 444 - */ 445 - if (atomic_dec_and_test(&core_state->nr_threads)) 446 - complete(&core_state->startup); 447 - 448 - for (;;) { 449 - set_current_state(TASK_IDLE|TASK_FREEZABLE); 450 - if (!self.task) /* see coredump_finish() */ 451 - break; 452 - schedule(); 453 - } 454 - __set_current_state(TASK_RUNNING); 435 + for (;;) { 436 + set_current_state(TASK_IDLE|TASK_FREEZABLE); 437 + if (!self.task) /* see coredump_finish() */ 438 + break; 439 + schedule(); 455 440 } 441 + __set_current_state(TASK_RUNNING); 456 442 } 457 443 458 444 #ifdef CONFIG_MEMCG ··· 862 876 { 863 877 struct sighand_struct *sighand = tsk->sighand; 864 878 struct signal_struct *signal = tsk->signal; 879 + struct core_state *core_state; 865 880 866 881 spin_lock_irq(&sighand->siglock); 867 882 signal->quick_threads--; ··· 872 885 signal->group_exit_code = code; 873 886 signal->group_stop_count = 0; 874 887 } 888 + /* 889 + * Serialize with any possible pending coredump. 890 + * We must hold siglock around checking core_state 891 + * and setting PF_POSTCOREDUMP. The core-inducing thread 892 + * will increment ->nr_threads for each thread in the 893 + * group without PF_POSTCOREDUMP set. 894 + */ 895 + tsk->flags |= PF_POSTCOREDUMP; 896 + core_state = signal->core_state; 875 897 spin_unlock_irq(&sighand->siglock); 898 + 899 + if (unlikely(core_state)) 900 + coredump_task_exit(tsk, core_state); 876 901 } 877 902 878 903 void __noreturn do_exit(long code) ··· 893 894 int group_dead; 894 895 895 896 WARN_ON(irqs_disabled()); 896 - 897 - synchronize_group_exit(tsk, code); 898 - 899 897 WARN_ON(tsk->plug); 900 898 901 899 kcov_task_exit(tsk); 902 900 kmsan_task_exit(tsk); 903 901 904 - coredump_task_exit(tsk); 902 + synchronize_group_exit(tsk, code); 905 903 ptrace_event(PTRACE_EVENT_EXIT, code); 906 904 user_events_exit(tsk); 907 905

Configure Feed

Configure Feed