printk: Wait for the global console lock when the system is going down

There are reports that the console kthreads block the global console
lock when the system is going down, for example, reboot, panic.

First part of the solution was to block kthreads in these problematic
system states so they stopped handling newly added messages.

Second part of the solution is to wait when for the kthreads when
they are actively printing. It solves the problem when a message
was printed before the system entered the problematic state and
the kthreads managed to step in.

A busy waiting has to be used because panic() can be called in any
context and in an unknown state of the scheduler.

There must be a timeout because the kthread might get stuck or sleeping
and never release the lock. The timeout 10s is an arbitrary value
inspired by the softlockup timeout.

Link: https://lore.kernel.org/r/20220610205038.GA3050413@paulmck-ThinkPad-P17-Gen-1
Link: https://lore.kernel.org/r/CAMdYzYpF4FNTBPZsEFeWRuEwSies36QM_As8osPWZSr2q-viEA@mail.gmail.com
Signed-off-by: Petr Mladek <pmladek@suse.com>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20220615162805.27962-3-pmladek@suse.com

Petr Mladek 4 years ago b87f0230 c3230283

+47

6 changed files

expand all

include

linux

printk.h

kernel

panic.c

printk

internal.h

printk.c

printk_safe.c

reboot.c

include/linux/printk.h

··· 174 174 extern void printk_prefer_direct_exit(void); 175 175 176 176 extern bool pr_flush(int timeout_ms, bool reset_on_progress); 177 + extern void try_block_console_kthreads(int timeout_ms); 177 178 178 179 /* 179 180 * Please don't use printk_ratelimit(), because it shares ratelimiting state ··· 237 236 static inline bool pr_flush(int timeout_ms, bool reset_on_progress) 238 237 { 239 238 return true; 239 + } 240 + 241 + static inline void try_block_console_kthreads(int timeout_ms) 242 + { 240 243 } 241 244 242 245 static inline int printk_ratelimit(void)

kernel/panic.c

··· 273 273 * unfortunately means it may not be hardened to work in a 274 274 * panic situation. 275 275 */ 276 + try_block_console_kthreads(10000); 276 277 smp_send_stop(); 277 278 } else { 278 279 /* ··· 281 280 * kmsg_dump, we will need architecture dependent extra 282 281 * works in addition to stopping other CPUs. 283 282 */ 283 + try_block_console_kthreads(10000); 284 284 crash_smp_send_stop(); 285 285 } 286 286

kernel/printk/internal.h

··· 20 20 LOG_CONT = 8, /* text is a fragment of a continuation line */ 21 21 }; 22 22 23 + extern bool block_console_kthreads; 24 + 23 25 __printf(4, 0) 24 26 int vprintk_store(int facility, int level, 25 27 const struct dev_printk_info *dev_info,

kernel/printk/printk.c

··· 250 250 #define console_kthread_printing_exit() \ 251 251 atomic_dec(&console_kthreads_active) 252 252 253 + /* Block console kthreads to avoid processing new messages. */ 254 + bool block_console_kthreads; 255 + 253 256 /* 254 257 * Helper macros to handle lockdep when locking/unlocking console_sem. We use 255 258 * macros instead of functions so that _RET_IP_ contains useful information. ··· 3733 3730 3734 3731 if (con->blocked || 3735 3732 console_kthreads_atomically_blocked() || 3733 + block_console_kthreads || 3736 3734 system_state > SYSTEM_RUNNING || 3737 3735 oops_in_progress) { 3738 3736 return false;

+32

kernel/printk/printk_safe.c

··· 8 8 #include <linux/smp.h> 9 9 #include <linux/cpumask.h> 10 10 #include <linux/printk.h> 11 + #include <linux/console.h> 11 12 #include <linux/kprobes.h> 13 + #include <linux/delay.h> 12 14 13 15 #include "internal.h" 14 16 ··· 52 50 return vprintk_default(fmt, args); 53 51 } 54 52 EXPORT_SYMBOL(vprintk); 53 + 54 + /** 55 + * try_block_console_kthreads() - Try to block console kthreads and 56 + * make the global console_lock() avaialble 57 + * 58 + * @timeout_ms: The maximum time (in ms) to wait. 59 + * 60 + * Prevent console kthreads from starting processing new messages. Wait 61 + * until the global console_lock() become available. 62 + * 63 + * Context: Can be called in any context. 64 + */ 65 + void try_block_console_kthreads(int timeout_ms) 66 + { 67 + block_console_kthreads = true; 68 + 69 + /* Do not wait when the console lock could not be safely taken. */ 70 + if (this_cpu_read(printk_context) || in_nmi()) 71 + return; 72 + 73 + while (timeout_ms > 0) { 74 + if (console_trylock()) { 75 + console_unlock(); 76 + return; 77 + } 78 + 79 + udelay(1000); 80 + timeout_ms -= 1; 81 + } 82 + }

kernel/reboot.c

··· 74 74 { 75 75 blocking_notifier_call_chain(&reboot_notifier_list, SYS_RESTART, cmd); 76 76 system_state = SYSTEM_RESTART; 77 + try_block_console_kthreads(10000); 77 78 usermodehelper_disable(); 78 79 device_shutdown(); 79 80 } ··· 263 262 blocking_notifier_call_chain(&reboot_notifier_list, 264 263 (state == SYSTEM_HALT) ? SYS_HALT : SYS_POWER_OFF, NULL); 265 264 system_state = state; 265 + try_block_console_kthreads(10000); 266 266 usermodehelper_disable(); 267 267 device_shutdown(); 268 268 }

Configure Feed

Configure Feed