Merge tag 'printk-for-6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/printk/linux

+129

include/linux/console.h

··· 156 156 * /dev/kmesg which requires a larger output buffer. 157 157 * @CON_SUSPENDED: Indicates if a console is suspended. If true, the 158 158 * printing callbacks must not be called. 159 + * @CON_NBCON: Console can operate outside of the legacy style console_lock 160 + * constraints. 159 161 */ 160 162 enum cons_flags { 161 163 CON_PRINTBUFFER = BIT(0), ··· 168 166 CON_BRL = BIT(5), 169 167 CON_EXTENDED = BIT(6), 170 168 CON_SUSPENDED = BIT(7), 169 + CON_NBCON = BIT(8), 170 + }; 171 + 172 + /** 173 + * struct nbcon_state - console state for nbcon consoles 174 + * @atom: Compound of the state fields for atomic operations 175 + * 176 + * @req_prio: The priority of a handover request 177 + * @prio: The priority of the current owner 178 + * @unsafe: Console is busy in a non takeover region 179 + * @unsafe_takeover: A hostile takeover in an unsafe state happened in the 180 + * past. The console cannot be safe until re-initialized. 181 + * @cpu: The CPU on which the owner runs 182 + * 183 + * To be used for reading and preparing of the value stored in the nbcon 184 + * state variable @console::nbcon_state. 185 + * 186 + * The @prio and @req_prio fields are particularly important to allow 187 + * spin-waiting to timeout and give up without the risk of a waiter being 188 + * assigned the lock after giving up. 189 + */ 190 + struct nbcon_state { 191 + union { 192 + unsigned int atom; 193 + struct { 194 + unsigned int prio : 2; 195 + unsigned int req_prio : 2; 196 + unsigned int unsafe : 1; 197 + unsigned int unsafe_takeover : 1; 198 + unsigned int cpu : 24; 199 + }; 200 + }; 201 + }; 202 + 203 + /* 204 + * The nbcon_state struct is used to easily create and interpret values that 205 + * are stored in the @console::nbcon_state variable. Ensure this struct stays 206 + * within the size boundaries of the atomic variable's underlying type in 207 + * order to avoid any accidental truncation. 208 + */ 209 + static_assert(sizeof(struct nbcon_state) <= sizeof(int)); 210 + 211 + /** 212 + * nbcon_prio - console owner priority for nbcon consoles 213 + * @NBCON_PRIO_NONE: Unused 214 + * @NBCON_PRIO_NORMAL: Normal (non-emergency) usage 215 + * @NBCON_PRIO_EMERGENCY: Emergency output (WARN/OOPS...) 216 + * @NBCON_PRIO_PANIC: Panic output 217 + * @NBCON_PRIO_MAX: The number of priority levels 218 + * 219 + * A higher priority context can takeover the console when it is 220 + * in the safe state. The final attempt to flush consoles in panic() 221 + * can be allowed to do so even in an unsafe state (Hope and pray). 222 + */ 223 + enum nbcon_prio { 224 + NBCON_PRIO_NONE = 0, 225 + NBCON_PRIO_NORMAL, 226 + NBCON_PRIO_EMERGENCY, 227 + NBCON_PRIO_PANIC, 228 + NBCON_PRIO_MAX, 229 + }; 230 + 231 + struct console; 232 + struct printk_buffers; 233 + 234 + /** 235 + * struct nbcon_context - Context for console acquire/release 236 + * @console: The associated console 237 + * @spinwait_max_us: Limit for spin-wait acquire 238 + * @prio: Priority of the context 239 + * @allow_unsafe_takeover: Allow performing takeover even if unsafe. Can 240 + * be used only with NBCON_PRIO_PANIC @prio. It 241 + * might cause a system freeze when the console 242 + * is used later. 243 + * @backlog: Ringbuffer has pending records 244 + * @pbufs: Pointer to the text buffer for this context 245 + * @seq: The sequence number to print for this context 246 + */ 247 + struct nbcon_context { 248 + /* members set by caller */ 249 + struct console *console; 250 + unsigned int spinwait_max_us; 251 + enum nbcon_prio prio; 252 + unsigned int allow_unsafe_takeover : 1; 253 + 254 + /* members set by emit */ 255 + unsigned int backlog : 1; 256 + 257 + /* members set by acquire */ 258 + struct printk_buffers *pbufs; 259 + u64 seq; 260 + }; 261 + 262 + /** 263 + * struct nbcon_write_context - Context handed to the nbcon write callbacks 264 + * @ctxt: The core console context 265 + * @outbuf: Pointer to the text buffer for output 266 + * @len: Length to write 267 + * @unsafe_takeover: If a hostile takeover in an unsafe state has occurred 268 + */ 269 + struct nbcon_write_context { 270 + struct nbcon_context __private ctxt; 271 + char *outbuf; 272 + unsigned int len; 273 + bool unsafe_takeover; 171 274 }; 172 275 173 276 /** ··· 294 187 * @dropped: Number of unreported dropped ringbuffer records 295 188 * @data: Driver private data 296 189 * @node: hlist node for the console list 190 + * 191 + * @write_atomic: Write callback for atomic context 192 + * @nbcon_state: State for nbcon consoles 193 + * @nbcon_seq: Sequence number of the next record for nbcon to print 194 + * @pbufs: Pointer to nbcon private buffer 297 195 */ 298 196 struct console { 299 197 char name[16]; ··· 318 206 unsigned long dropped; 319 207 void *data; 320 208 struct hlist_node node; 209 + 210 + /* nbcon console specific members */ 211 + bool (*write_atomic)(struct console *con, 212 + struct nbcon_write_context *wctxt); 213 + atomic_t __private nbcon_state; 214 + atomic_long_t __private nbcon_seq; 215 + struct printk_buffers *pbufs; 321 216 }; 322 217 323 218 #ifdef CONFIG_LOCKDEP ··· 450 331 #define for_each_console(con) \ 451 332 lockdep_assert_console_list_lock_held(); \ 452 333 hlist_for_each_entry(con, &console_list, node) 334 + 335 + #ifdef CONFIG_PRINTK 336 + extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt); 337 + extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt); 338 + extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt); 339 + #else 340 + static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; } 341 + static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; } 342 + static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; } 343 + #endif 453 344 454 345 extern int console_set_on_cmdline; 455 346 extern struct console *early_console;

+1 -1

kernel/printk/Makefile

··· 1 1 # SPDX-License-Identifier: GPL-2.0-only 2 2 obj-y = printk.o 3 - obj-$(CONFIG_PRINTK) += printk_safe.o 3 + obj-$(CONFIG_PRINTK) += printk_safe.o nbcon.o 4 4 obj-$(CONFIG_A11Y_BRAILLE_CONSOLE) += braille.o 5 5 obj-$(CONFIG_PRINTK_INDEX) += index.o 6 6

+31

kernel/printk/internal.h

··· 3 3 * internal.h - printk internal definitions 4 4 */ 5 5 #include <linux/percpu.h> 6 + #include <linux/console.h> 7 + #include "printk_ringbuffer.h" 6 8 7 9 #if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL) 8 10 void __init printk_sysctl_init(void); ··· 13 11 #else 14 12 #define printk_sysctl_init() do { } while (0) 15 13 #endif 14 + 15 + #define con_printk(lvl, con, fmt, ...) \ 16 + printk(lvl pr_fmt("%s%sconsole [%s%d] " fmt), \ 17 + (con->flags & CON_NBCON) ? "" : "legacy ", \ 18 + (con->flags & CON_BOOT) ? "boot" : "", \ 19 + con->name, con->index, ##__VA_ARGS__) 16 20 17 21 #ifdef CONFIG_PRINTK 18 22 ··· 42 34 LOG_NEWLINE = 2, /* text ended with a newline */ 43 35 LOG_CONT = 8, /* text is a fragment of a continuation line */ 44 36 }; 37 + 38 + extern struct printk_ringbuffer *prb; 45 39 46 40 __printf(4, 0) 47 41 int vprintk_store(int facility, int level, ··· 71 61 72 62 u16 printk_parse_prefix(const char *text, int *level, 73 63 enum printk_info_flags *flags); 64 + 65 + u64 nbcon_seq_read(struct console *con); 66 + void nbcon_seq_force(struct console *con, u64 seq); 67 + bool nbcon_alloc(struct console *con); 68 + void nbcon_init(struct console *con); 69 + void nbcon_free(struct console *con); 70 + 74 71 #else 75 72 76 73 #define PRINTK_PREFIX_MAX 0 ··· 93 76 #define printk_safe_exit_irqrestore(flags) local_irq_restore(flags) 94 77 95 78 static inline bool printk_percpu_data_ready(void) { return false; } 79 + static inline u64 nbcon_seq_read(struct console *con) { return 0; } 80 + static inline void nbcon_seq_force(struct console *con, u64 seq) { } 81 + static inline bool nbcon_alloc(struct console *con) { return false; } 82 + static inline void nbcon_init(struct console *con) { } 83 + static inline void nbcon_free(struct console *con) { } 84 + 96 85 #endif /* CONFIG_PRINTK */ 86 + 87 + extern struct printk_buffers printk_shared_pbufs; 97 88 98 89 /** 99 90 * struct printk_buffers - Buffers to read/format/output printk messages. ··· 130 105 }; 131 106 132 107 bool other_cpu_in_panic(void); 108 + bool printk_get_next_message(struct printk_message *pmsg, u64 seq, 109 + bool is_extended, bool may_supress); 110 + 111 + #ifdef CONFIG_PRINTK 112 + void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped); 113 + #endif

+1029

kernel/printk/nbcon.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + // Copyright (C) 2022 Linutronix GmbH, John Ogness 3 + // Copyright (C) 2022 Intel, Thomas Gleixner 4 + 5 + #include <linux/kernel.h> 6 + #include <linux/console.h> 7 + #include <linux/delay.h> 8 + #include <linux/slab.h> 9 + #include "internal.h" 10 + /* 11 + * Printk console printing implementation for consoles which does not depend 12 + * on the legacy style console_lock mechanism. 13 + * 14 + * The state of the console is maintained in the "nbcon_state" atomic 15 + * variable. 16 + * 17 + * The console is locked when: 18 + * 19 + * - The 'prio' field contains the priority of the context that owns the 20 + * console. Only higher priority contexts are allowed to take over the 21 + * lock. A value of 0 (NBCON_PRIO_NONE) means the console is not locked. 22 + * 23 + * - The 'cpu' field denotes on which CPU the console is locked. It is used 24 + * to prevent busy waiting on the same CPU. Also it informs the lock owner 25 + * that it has lost the lock in a more complex scenario when the lock was 26 + * taken over by a higher priority context, released, and taken on another 27 + * CPU with the same priority as the interrupted owner. 28 + * 29 + * The acquire mechanism uses a few more fields: 30 + * 31 + * - The 'req_prio' field is used by the handover approach to make the 32 + * current owner aware that there is a context with a higher priority 33 + * waiting for the friendly handover. 34 + * 35 + * - The 'unsafe' field allows to take over the console in a safe way in the 36 + * middle of emitting a message. The field is set only when accessing some 37 + * shared resources or when the console device is manipulated. It can be 38 + * cleared, for example, after emitting one character when the console 39 + * device is in a consistent state. 40 + * 41 + * - The 'unsafe_takeover' field is set when a hostile takeover took the 42 + * console in an unsafe state. The console will stay in the unsafe state 43 + * until re-initialized. 44 + * 45 + * The acquire mechanism uses three approaches: 46 + * 47 + * 1) Direct acquire when the console is not owned or is owned by a lower 48 + * priority context and is in a safe state. 49 + * 50 + * 2) Friendly handover mechanism uses a request/grant handshake. It is used 51 + * when the current owner has lower priority and the console is in an 52 + * unsafe state. 53 + * 54 + * The requesting context: 55 + * 56 + * a) Sets its priority into the 'req_prio' field. 57 + * 58 + * b) Waits (with a timeout) for the owning context to unlock the 59 + * console. 60 + * 61 + * c) Takes the lock and clears the 'req_prio' field. 62 + * 63 + * The owning context: 64 + * 65 + * a) Observes the 'req_prio' field set on exit from the unsafe 66 + * console state. 67 + * 68 + * b) Gives up console ownership by clearing the 'prio' field. 69 + * 70 + * 3) Unsafe hostile takeover allows to take over the lock even when the 71 + * console is an unsafe state. It is used only in panic() by the final 72 + * attempt to flush consoles in a try and hope mode. 73 + * 74 + * Note that separate record buffers are used in panic(). As a result, 75 + * the messages can be read and formatted without any risk even after 76 + * using the hostile takeover in unsafe state. 77 + * 78 + * The release function simply clears the 'prio' field. 79 + * 80 + * All operations on @console::nbcon_state are atomic cmpxchg based to 81 + * handle concurrency. 82 + * 83 + * The acquire/release functions implement only minimal policies: 84 + * 85 + * - Preference for higher priority contexts. 86 + * - Protection of the panic CPU. 87 + * 88 + * All other policy decisions must be made at the call sites: 89 + * 90 + * - What is marked as an unsafe section. 91 + * - Whether to spin-wait if there is already an owner and the console is 92 + * in an unsafe state. 93 + * - Whether to attempt an unsafe hostile takeover. 94 + * 95 + * The design allows to implement the well known: 96 + * 97 + * acquire() 98 + * output_one_printk_record() 99 + * release() 100 + * 101 + * The output of one printk record might be interrupted with a higher priority 102 + * context. The new owner is supposed to reprint the entire interrupted record 103 + * from scratch. 104 + */ 105 + 106 + /** 107 + * nbcon_state_set - Helper function to set the console state 108 + * @con: Console to update 109 + * @new: The new state to write 110 + * 111 + * Only to be used when the console is not yet or no longer visible in the 112 + * system. Otherwise use nbcon_state_try_cmpxchg(). 113 + */ 114 + static inline void nbcon_state_set(struct console *con, struct nbcon_state *new) 115 + { 116 + atomic_set(&ACCESS_PRIVATE(con, nbcon_state), new->atom); 117 + } 118 + 119 + /** 120 + * nbcon_state_read - Helper function to read the console state 121 + * @con: Console to read 122 + * @state: The state to store the result 123 + */ 124 + static inline void nbcon_state_read(struct console *con, struct nbcon_state *state) 125 + { 126 + state->atom = atomic_read(&ACCESS_PRIVATE(con, nbcon_state)); 127 + } 128 + 129 + /** 130 + * nbcon_state_try_cmpxchg() - Helper function for atomic_try_cmpxchg() on console state 131 + * @con: Console to update 132 + * @cur: Old/expected state 133 + * @new: New state 134 + * 135 + * Return: True on success. False on fail and @cur is updated. 136 + */ 137 + static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_state *cur, 138 + struct nbcon_state *new) 139 + { 140 + return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom); 141 + } 142 + 143 + #ifdef CONFIG_64BIT 144 + 145 + #define __seq_to_nbcon_seq(seq) (seq) 146 + #define __nbcon_seq_to_seq(seq) (seq) 147 + 148 + #else /* CONFIG_64BIT */ 149 + 150 + #define __seq_to_nbcon_seq(seq) ((u32)seq) 151 + 152 + static inline u64 __nbcon_seq_to_seq(u32 nbcon_seq) 153 + { 154 + u64 seq; 155 + u64 rb_next_seq; 156 + 157 + /* 158 + * The provided sequence is only the lower 32 bits of the ringbuffer 159 + * sequence. It needs to be expanded to 64bit. Get the next sequence 160 + * number from the ringbuffer and fold it. 161 + * 162 + * Having a 32bit representation in the console is sufficient. 163 + * If a console ever gets more than 2^31 records behind 164 + * the ringbuffer then this is the least of the problems. 165 + * 166 + * Also the access to the ring buffer is always safe. 167 + */ 168 + rb_next_seq = prb_next_seq(prb); 169 + seq = rb_next_seq - ((u32)rb_next_seq - nbcon_seq); 170 + 171 + return seq; 172 + } 173 + 174 + #endif /* CONFIG_64BIT */ 175 + 176 + /** 177 + * nbcon_seq_read - Read the current console sequence 178 + * @con: Console to read the sequence of 179 + * 180 + * Return: Sequence number of the next record to print on @con. 181 + */ 182 + u64 nbcon_seq_read(struct console *con) 183 + { 184 + unsigned long nbcon_seq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_seq)); 185 + 186 + return __nbcon_seq_to_seq(nbcon_seq); 187 + } 188 + 189 + /** 190 + * nbcon_seq_force - Force console sequence to a specific value 191 + * @con: Console to work on 192 + * @seq: Sequence number value to set 193 + * 194 + * Only to be used during init (before registration) or in extreme situations 195 + * (such as panic with CONSOLE_REPLAY_ALL). 196 + */ 197 + void nbcon_seq_force(struct console *con, u64 seq) 198 + { 199 + /* 200 + * If the specified record no longer exists, the oldest available record 201 + * is chosen. This is especially important on 32bit systems because only 202 + * the lower 32 bits of the sequence number are stored. The upper 32 bits 203 + * are derived from the sequence numbers available in the ringbuffer. 204 + */ 205 + u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb)); 206 + 207 + atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __seq_to_nbcon_seq(valid_seq)); 208 + 209 + /* Clear con->seq since nbcon consoles use con->nbcon_seq instead. */ 210 + con->seq = 0; 211 + } 212 + 213 + /** 214 + * nbcon_seq_try_update - Try to update the console sequence number 215 + * @ctxt: Pointer to an acquire context that contains 216 + * all information about the acquire mode 217 + * @new_seq: The new sequence number to set 218 + * 219 + * @ctxt->seq is updated to the new value of @con::nbcon_seq (expanded to 220 + * the 64bit value). This could be a different value than @new_seq if 221 + * nbcon_seq_force() was used or the current context no longer owns the 222 + * console. In the later case, it will stop printing anyway. 223 + */ 224 + static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq) 225 + { 226 + unsigned long nbcon_seq = __seq_to_nbcon_seq(ctxt->seq); 227 + struct console *con = ctxt->console; 228 + 229 + if (atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_seq), &nbcon_seq, 230 + __seq_to_nbcon_seq(new_seq))) { 231 + ctxt->seq = new_seq; 232 + } else { 233 + ctxt->seq = nbcon_seq_read(con); 234 + } 235 + } 236 + 237 + /** 238 + * nbcon_context_try_acquire_direct - Try to acquire directly 239 + * @ctxt: The context of the caller 240 + * @cur: The current console state 241 + * 242 + * Acquire the console when it is released. Also acquire the console when 243 + * the current owner has a lower priority and the console is in a safe state. 244 + * 245 + * Return: 0 on success. Otherwise, an error code on failure. Also @cur 246 + * is updated to the latest state when failed to modify it. 247 + * 248 + * Errors: 249 + * 250 + * -EPERM: A panic is in progress and this is not the panic CPU. 251 + * Or the current owner or waiter has the same or higher 252 + * priority. No acquire method can be successful in 253 + * this case. 254 + * 255 + * -EBUSY: The current owner has a lower priority but the console 256 + * in an unsafe state. The caller should try using 257 + * the handover acquire method. 258 + */ 259 + static int nbcon_context_try_acquire_direct(struct nbcon_context *ctxt, 260 + struct nbcon_state *cur) 261 + { 262 + unsigned int cpu = smp_processor_id(); 263 + struct console *con = ctxt->console; 264 + struct nbcon_state new; 265 + 266 + do { 267 + if (other_cpu_in_panic()) 268 + return -EPERM; 269 + 270 + if (ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio) 271 + return -EPERM; 272 + 273 + if (cur->unsafe) 274 + return -EBUSY; 275 + 276 + /* 277 + * The console should never be safe for a direct acquire 278 + * if an unsafe hostile takeover has ever happened. 279 + */ 280 + WARN_ON_ONCE(cur->unsafe_takeover); 281 + 282 + new.atom = cur->atom; 283 + new.prio = ctxt->prio; 284 + new.req_prio = NBCON_PRIO_NONE; 285 + new.unsafe = cur->unsafe_takeover; 286 + new.cpu = cpu; 287 + 288 + } while (!nbcon_state_try_cmpxchg(con, cur, &new)); 289 + 290 + return 0; 291 + } 292 + 293 + static bool nbcon_waiter_matches(struct nbcon_state *cur, int expected_prio) 294 + { 295 + /* 296 + * The request context is well defined by the @req_prio because: 297 + * 298 + * - Only a context with a higher priority can take over the request. 299 + * - There are only three priorities. 300 + * - Only one CPU is allowed to request PANIC priority. 301 + * - Lower priorities are ignored during panic() until reboot. 302 + * 303 + * As a result, the following scenario is *not* possible: 304 + * 305 + * 1. Another context with a higher priority directly takes ownership. 306 + * 2. The higher priority context releases the ownership. 307 + * 3. A lower priority context takes the ownership. 308 + * 4. Another context with the same priority as this context 309 + * creates a request and starts waiting. 310 + */ 311 + 312 + return (cur->req_prio == expected_prio); 313 + } 314 + 315 + /** 316 + * nbcon_context_try_acquire_requested - Try to acquire after having 317 + * requested a handover 318 + * @ctxt: The context of the caller 319 + * @cur: The current console state 320 + * 321 + * This is a helper function for nbcon_context_try_acquire_handover(). 322 + * It is called when the console is in an unsafe state. The current 323 + * owner will release the console on exit from the unsafe region. 324 + * 325 + * Return: 0 on success and @cur is updated to the new console state. 326 + * Otherwise an error code on failure. 327 + * 328 + * Errors: 329 + * 330 + * -EPERM: A panic is in progress and this is not the panic CPU 331 + * or this context is no longer the waiter. 332 + * 333 + * -EBUSY: The console is still locked. The caller should 334 + * continue waiting. 335 + * 336 + * Note: The caller must still remove the request when an error has occurred 337 + * except when this context is no longer the waiter. 338 + */ 339 + static int nbcon_context_try_acquire_requested(struct nbcon_context *ctxt, 340 + struct nbcon_state *cur) 341 + { 342 + unsigned int cpu = smp_processor_id(); 343 + struct console *con = ctxt->console; 344 + struct nbcon_state new; 345 + 346 + /* Note that the caller must still remove the request! */ 347 + if (other_cpu_in_panic()) 348 + return -EPERM; 349 + 350 + /* 351 + * Note that the waiter will also change if there was an unsafe 352 + * hostile takeover. 353 + */ 354 + if (!nbcon_waiter_matches(cur, ctxt->prio)) 355 + return -EPERM; 356 + 357 + /* If still locked, caller should continue waiting. */ 358 + if (cur->prio != NBCON_PRIO_NONE) 359 + return -EBUSY; 360 + 361 + /* 362 + * The previous owner should have never released ownership 363 + * in an unsafe region. 364 + */ 365 + WARN_ON_ONCE(cur->unsafe); 366 + 367 + new.atom = cur->atom; 368 + new.prio = ctxt->prio; 369 + new.req_prio = NBCON_PRIO_NONE; 370 + new.unsafe = cur->unsafe_takeover; 371 + new.cpu = cpu; 372 + 373 + if (!nbcon_state_try_cmpxchg(con, cur, &new)) { 374 + /* 375 + * The acquire could fail only when it has been taken 376 + * over by a higher priority context. 377 + */ 378 + WARN_ON_ONCE(nbcon_waiter_matches(cur, ctxt->prio)); 379 + return -EPERM; 380 + } 381 + 382 + /* Handover success. This context now owns the console. */ 383 + return 0; 384 + } 385 + 386 + /** 387 + * nbcon_context_try_acquire_handover - Try to acquire via handover 388 + * @ctxt: The context of the caller 389 + * @cur: The current console state 390 + * 391 + * The function must be called only when the context has higher priority 392 + * than the current owner and the console is in an unsafe state. 393 + * It is the case when nbcon_context_try_acquire_direct() returns -EBUSY. 394 + * 395 + * The function sets "req_prio" field to make the current owner aware of 396 + * the request. Then it waits until the current owner releases the console, 397 + * or an even higher context takes over the request, or timeout expires. 398 + * 399 + * The current owner checks the "req_prio" field on exit from the unsafe 400 + * region and releases the console. It does not touch the "req_prio" field 401 + * so that the console stays reserved for the waiter. 402 + * 403 + * Return: 0 on success. Otherwise, an error code on failure. Also @cur 404 + * is updated to the latest state when failed to modify it. 405 + * 406 + * Errors: 407 + * 408 + * -EPERM: A panic is in progress and this is not the panic CPU. 409 + * Or a higher priority context has taken over the 410 + * console or the handover request. 411 + * 412 + * -EBUSY: The current owner is on the same CPU so that the hand 413 + * shake could not work. Or the current owner is not 414 + * willing to wait (zero timeout). Or the console does 415 + * not enter the safe state before timeout passed. The 416 + * caller might still use the unsafe hostile takeover 417 + * when allowed. 418 + * 419 + * -EAGAIN: @cur has changed when creating the handover request. 420 + * The caller should retry with direct acquire. 421 + */ 422 + static int nbcon_context_try_acquire_handover(struct nbcon_context *ctxt, 423 + struct nbcon_state *cur) 424 + { 425 + unsigned int cpu = smp_processor_id(); 426 + struct console *con = ctxt->console; 427 + struct nbcon_state new; 428 + int timeout; 429 + int request_err = -EBUSY; 430 + 431 + /* 432 + * Check that the handover is called when the direct acquire failed 433 + * with -EBUSY. 434 + */ 435 + WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio); 436 + WARN_ON_ONCE(!cur->unsafe); 437 + 438 + /* Handover is not possible on the same CPU. */ 439 + if (cur->cpu == cpu) 440 + return -EBUSY; 441 + 442 + /* 443 + * Console stays unsafe after an unsafe takeover until re-initialized. 444 + * Waiting is not going to help in this case. 445 + */ 446 + if (cur->unsafe_takeover) 447 + return -EBUSY; 448 + 449 + /* Is the caller willing to wait? */ 450 + if (ctxt->spinwait_max_us == 0) 451 + return -EBUSY; 452 + 453 + /* 454 + * Setup a request for the handover. The caller should try to acquire 455 + * the console directly when the current state has been modified. 456 + */ 457 + new.atom = cur->atom; 458 + new.req_prio = ctxt->prio; 459 + if (!nbcon_state_try_cmpxchg(con, cur, &new)) 460 + return -EAGAIN; 461 + 462 + cur->atom = new.atom; 463 + 464 + /* Wait until there is no owner and then acquire the console. */ 465 + for (timeout = ctxt->spinwait_max_us; timeout >= 0; timeout--) { 466 + /* On successful acquire, this request is cleared. */ 467 + request_err = nbcon_context_try_acquire_requested(ctxt, cur); 468 + if (!request_err) 469 + return 0; 470 + 471 + /* 472 + * If the acquire should be aborted, it must be ensured 473 + * that the request is removed before returning to caller. 474 + */ 475 + if (request_err == -EPERM) 476 + break; 477 + 478 + udelay(1); 479 + 480 + /* Re-read the state because some time has passed. */ 481 + nbcon_state_read(con, cur); 482 + } 483 + 484 + /* Timed out or aborted. Carefully remove handover request. */ 485 + do { 486 + /* 487 + * No need to remove request if there is a new waiter. This 488 + * can only happen if a higher priority context has taken over 489 + * the console or the handover request. 490 + */ 491 + if (!nbcon_waiter_matches(cur, ctxt->prio)) 492 + return -EPERM; 493 + 494 + /* Unset request for handover. */ 495 + new.atom = cur->atom; 496 + new.req_prio = NBCON_PRIO_NONE; 497 + if (nbcon_state_try_cmpxchg(con, cur, &new)) { 498 + /* 499 + * Request successfully unset. Report failure of 500 + * acquiring via handover. 501 + */ 502 + cur->atom = new.atom; 503 + return request_err; 504 + } 505 + 506 + /* 507 + * Unable to remove request. Try to acquire in case 508 + * the owner has released the lock. 509 + */ 510 + } while (nbcon_context_try_acquire_requested(ctxt, cur)); 511 + 512 + /* Lucky timing. The acquire succeeded while removing the request. */ 513 + return 0; 514 + } 515 + 516 + /** 517 + * nbcon_context_try_acquire_hostile - Acquire via unsafe hostile takeover 518 + * @ctxt: The context of the caller 519 + * @cur: The current console state 520 + * 521 + * Acquire the console even in the unsafe state. 522 + * 523 + * It can be permitted by setting the 'allow_unsafe_takeover' field only 524 + * by the final attempt to flush messages in panic(). 525 + * 526 + * Return: 0 on success. -EPERM when not allowed by the context. 527 + */ 528 + static int nbcon_context_try_acquire_hostile(struct nbcon_context *ctxt, 529 + struct nbcon_state *cur) 530 + { 531 + unsigned int cpu = smp_processor_id(); 532 + struct console *con = ctxt->console; 533 + struct nbcon_state new; 534 + 535 + if (!ctxt->allow_unsafe_takeover) 536 + return -EPERM; 537 + 538 + /* Ensure caller is allowed to perform unsafe hostile takeovers. */ 539 + if (WARN_ON_ONCE(ctxt->prio != NBCON_PRIO_PANIC)) 540 + return -EPERM; 541 + 542 + /* 543 + * Check that try_acquire_direct() and try_acquire_handover() returned 544 + * -EBUSY in the right situation. 545 + */ 546 + WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio); 547 + WARN_ON_ONCE(cur->unsafe != true); 548 + 549 + do { 550 + new.atom = cur->atom; 551 + new.cpu = cpu; 552 + new.prio = ctxt->prio; 553 + new.unsafe |= cur->unsafe_takeover; 554 + new.unsafe_takeover |= cur->unsafe; 555 + 556 + } while (!nbcon_state_try_cmpxchg(con, cur, &new)); 557 + 558 + return 0; 559 + } 560 + 561 + static struct printk_buffers panic_nbcon_pbufs; 562 + 563 + /** 564 + * nbcon_context_try_acquire - Try to acquire nbcon console 565 + * @ctxt: The context of the caller 566 + * 567 + * Return: True if the console was acquired. False otherwise. 568 + * 569 + * If the caller allowed an unsafe hostile takeover, on success the 570 + * caller should check the current console state to see if it is 571 + * in an unsafe state. Otherwise, on success the caller may assume 572 + * the console is not in an unsafe state. 573 + */ 574 + __maybe_unused 575 + static bool nbcon_context_try_acquire(struct nbcon_context *ctxt) 576 + { 577 + unsigned int cpu = smp_processor_id(); 578 + struct console *con = ctxt->console; 579 + struct nbcon_state cur; 580 + int err; 581 + 582 + nbcon_state_read(con, &cur); 583 + try_again: 584 + err = nbcon_context_try_acquire_direct(ctxt, &cur); 585 + if (err != -EBUSY) 586 + goto out; 587 + 588 + err = nbcon_context_try_acquire_handover(ctxt, &cur); 589 + if (err == -EAGAIN) 590 + goto try_again; 591 + if (err != -EBUSY) 592 + goto out; 593 + 594 + err = nbcon_context_try_acquire_hostile(ctxt, &cur); 595 + out: 596 + if (err) 597 + return false; 598 + 599 + /* Acquire succeeded. */ 600 + 601 + /* Assign the appropriate buffer for this context. */ 602 + if (atomic_read(&panic_cpu) == cpu) 603 + ctxt->pbufs = &panic_nbcon_pbufs; 604 + else 605 + ctxt->pbufs = con->pbufs; 606 + 607 + /* Set the record sequence for this context to print. */ 608 + ctxt->seq = nbcon_seq_read(ctxt->console); 609 + 610 + return true; 611 + } 612 + 613 + static bool nbcon_owner_matches(struct nbcon_state *cur, int expected_cpu, 614 + int expected_prio) 615 + { 616 + /* 617 + * Since consoles can only be acquired by higher priorities, 618 + * owning contexts are uniquely identified by @prio. However, 619 + * since contexts can unexpectedly lose ownership, it is 620 + * possible that later another owner appears with the same 621 + * priority. For this reason @cpu is also needed. 622 + */ 623 + 624 + if (cur->prio != expected_prio) 625 + return false; 626 + 627 + if (cur->cpu != expected_cpu) 628 + return false; 629 + 630 + return true; 631 + } 632 + 633 + /** 634 + * nbcon_context_release - Release the console 635 + * @ctxt: The nbcon context from nbcon_context_try_acquire() 636 + */ 637 + static void nbcon_context_release(struct nbcon_context *ctxt) 638 + { 639 + unsigned int cpu = smp_processor_id(); 640 + struct console *con = ctxt->console; 641 + struct nbcon_state cur; 642 + struct nbcon_state new; 643 + 644 + nbcon_state_read(con, &cur); 645 + 646 + do { 647 + if (!nbcon_owner_matches(&cur, cpu, ctxt->prio)) 648 + break; 649 + 650 + new.atom = cur.atom; 651 + new.prio = NBCON_PRIO_NONE; 652 + 653 + /* 654 + * If @unsafe_takeover is set, it is kept set so that 655 + * the state remains permanently unsafe. 656 + */ 657 + new.unsafe |= cur.unsafe_takeover; 658 + 659 + } while (!nbcon_state_try_cmpxchg(con, &cur, &new)); 660 + 661 + ctxt->pbufs = NULL; 662 + } 663 + 664 + /** 665 + * nbcon_context_can_proceed - Check whether ownership can proceed 666 + * @ctxt: The nbcon context from nbcon_context_try_acquire() 667 + * @cur: The current console state 668 + * 669 + * Return: True if this context still owns the console. False if 670 + * ownership was handed over or taken. 671 + * 672 + * Must be invoked when entering the unsafe state to make sure that it still 673 + * owns the lock. Also must be invoked when exiting the unsafe context 674 + * to eventually free the lock for a higher priority context which asked 675 + * for the friendly handover. 676 + * 677 + * It can be called inside an unsafe section when the console is just 678 + * temporary in safe state instead of exiting and entering the unsafe 679 + * state. 680 + * 681 + * Also it can be called in the safe context before doing an expensive 682 + * safe operation. It does not make sense to do the operation when 683 + * a higher priority context took the lock. 684 + * 685 + * When this function returns false then the calling context no longer owns 686 + * the console and is no longer allowed to go forward. In this case it must 687 + * back out immediately and carefully. The buffer content is also no longer 688 + * trusted since it no longer belongs to the calling context. 689 + */ 690 + static bool nbcon_context_can_proceed(struct nbcon_context *ctxt, struct nbcon_state *cur) 691 + { 692 + unsigned int cpu = smp_processor_id(); 693 + 694 + /* Make sure this context still owns the console. */ 695 + if (!nbcon_owner_matches(cur, cpu, ctxt->prio)) 696 + return false; 697 + 698 + /* The console owner can proceed if there is no waiter. */ 699 + if (cur->req_prio == NBCON_PRIO_NONE) 700 + return true; 701 + 702 + /* 703 + * A console owner within an unsafe region is always allowed to 704 + * proceed, even if there are waiters. It can perform a handover 705 + * when exiting the unsafe region. Otherwise the waiter will 706 + * need to perform an unsafe hostile takeover. 707 + */ 708 + if (cur->unsafe) 709 + return true; 710 + 711 + /* Waiters always have higher priorities than owners. */ 712 + WARN_ON_ONCE(cur->req_prio <= cur->prio); 713 + 714 + /* 715 + * Having a safe point for take over and eventually a few 716 + * duplicated characters or a full line is way better than a 717 + * hostile takeover. Post processing can take care of the garbage. 718 + * Release and hand over. 719 + */ 720 + nbcon_context_release(ctxt); 721 + 722 + /* 723 + * It is not clear whether the waiter really took over ownership. The 724 + * outermost callsite must make the final decision whether console 725 + * ownership is needed for it to proceed. If yes, it must reacquire 726 + * ownership (possibly hostile) before carefully proceeding. 727 + * 728 + * The calling context no longer owns the console so go back all the 729 + * way instead of trying to implement reacquire heuristics in tons of 730 + * places. 731 + */ 732 + return false; 733 + } 734 + 735 + /** 736 + * nbcon_can_proceed - Check whether ownership can proceed 737 + * @wctxt: The write context that was handed to the write function 738 + * 739 + * Return: True if this context still owns the console. False if 740 + * ownership was handed over or taken. 741 + * 742 + * It is used in nbcon_enter_unsafe() to make sure that it still owns the 743 + * lock. Also it is used in nbcon_exit_unsafe() to eventually free the lock 744 + * for a higher priority context which asked for the friendly handover. 745 + * 746 + * It can be called inside an unsafe section when the console is just 747 + * temporary in safe state instead of exiting and entering the unsafe state. 748 + * 749 + * Also it can be called in the safe context before doing an expensive safe 750 + * operation. It does not make sense to do the operation when a higher 751 + * priority context took the lock. 752 + * 753 + * When this function returns false then the calling context no longer owns 754 + * the console and is no longer allowed to go forward. In this case it must 755 + * back out immediately and carefully. The buffer content is also no longer 756 + * trusted since it no longer belongs to the calling context. 757 + */ 758 + bool nbcon_can_proceed(struct nbcon_write_context *wctxt) 759 + { 760 + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); 761 + struct console *con = ctxt->console; 762 + struct nbcon_state cur; 763 + 764 + nbcon_state_read(con, &cur); 765 + 766 + return nbcon_context_can_proceed(ctxt, &cur); 767 + } 768 + EXPORT_SYMBOL_GPL(nbcon_can_proceed); 769 + 770 + #define nbcon_context_enter_unsafe(c) __nbcon_context_update_unsafe(c, true) 771 + #define nbcon_context_exit_unsafe(c) __nbcon_context_update_unsafe(c, false) 772 + 773 + /** 774 + * __nbcon_context_update_unsafe - Update the unsafe bit in @con->nbcon_state 775 + * @ctxt: The nbcon context from nbcon_context_try_acquire() 776 + * @unsafe: The new value for the unsafe bit 777 + * 778 + * Return: True if the unsafe state was updated and this context still 779 + * owns the console. Otherwise false if ownership was handed 780 + * over or taken. 781 + * 782 + * This function allows console owners to modify the unsafe status of the 783 + * console. 784 + * 785 + * When this function returns false then the calling context no longer owns 786 + * the console and is no longer allowed to go forward. In this case it must 787 + * back out immediately and carefully. The buffer content is also no longer 788 + * trusted since it no longer belongs to the calling context. 789 + * 790 + * Internal helper to avoid duplicated code. 791 + */ 792 + static bool __nbcon_context_update_unsafe(struct nbcon_context *ctxt, bool unsafe) 793 + { 794 + struct console *con = ctxt->console; 795 + struct nbcon_state cur; 796 + struct nbcon_state new; 797 + 798 + nbcon_state_read(con, &cur); 799 + 800 + do { 801 + /* 802 + * The unsafe bit must not be cleared if an 803 + * unsafe hostile takeover has occurred. 804 + */ 805 + if (!unsafe && cur.unsafe_takeover) 806 + goto out; 807 + 808 + if (!nbcon_context_can_proceed(ctxt, &cur)) 809 + return false; 810 + 811 + new.atom = cur.atom; 812 + new.unsafe = unsafe; 813 + } while (!nbcon_state_try_cmpxchg(con, &cur, &new)); 814 + 815 + cur.atom = new.atom; 816 + out: 817 + return nbcon_context_can_proceed(ctxt, &cur); 818 + } 819 + 820 + /** 821 + * nbcon_enter_unsafe - Enter an unsafe region in the driver 822 + * @wctxt: The write context that was handed to the write function 823 + * 824 + * Return: True if this context still owns the console. False if 825 + * ownership was handed over or taken. 826 + * 827 + * When this function returns false then the calling context no longer owns 828 + * the console and is no longer allowed to go forward. In this case it must 829 + * back out immediately and carefully. The buffer content is also no longer 830 + * trusted since it no longer belongs to the calling context. 831 + */ 832 + bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) 833 + { 834 + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); 835 + 836 + return nbcon_context_enter_unsafe(ctxt); 837 + } 838 + EXPORT_SYMBOL_GPL(nbcon_enter_unsafe); 839 + 840 + /** 841 + * nbcon_exit_unsafe - Exit an unsafe region in the driver 842 + * @wctxt: The write context that was handed to the write function 843 + * 844 + * Return: True if this context still owns the console. False if 845 + * ownership was handed over or taken. 846 + * 847 + * When this function returns false then the calling context no longer owns 848 + * the console and is no longer allowed to go forward. In this case it must 849 + * back out immediately and carefully. The buffer content is also no longer 850 + * trusted since it no longer belongs to the calling context. 851 + */ 852 + bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) 853 + { 854 + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); 855 + 856 + return nbcon_context_exit_unsafe(ctxt); 857 + } 858 + EXPORT_SYMBOL_GPL(nbcon_exit_unsafe); 859 + 860 + /** 861 + * nbcon_emit_next_record - Emit a record in the acquired context 862 + * @wctxt: The write context that will be handed to the write function 863 + * 864 + * Return: True if this context still owns the console. False if 865 + * ownership was handed over or taken. 866 + * 867 + * When this function returns false then the calling context no longer owns 868 + * the console and is no longer allowed to go forward. In this case it must 869 + * back out immediately and carefully. The buffer content is also no longer 870 + * trusted since it no longer belongs to the calling context. If the caller 871 + * wants to do more it must reacquire the console first. 872 + * 873 + * When true is returned, @wctxt->ctxt.backlog indicates whether there are 874 + * still records pending in the ringbuffer, 875 + */ 876 + __maybe_unused 877 + static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) 878 + { 879 + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); 880 + struct console *con = ctxt->console; 881 + bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED; 882 + struct printk_message pmsg = { 883 + .pbufs = ctxt->pbufs, 884 + }; 885 + unsigned long con_dropped; 886 + struct nbcon_state cur; 887 + unsigned long dropped; 888 + bool done; 889 + 890 + /* 891 + * The printk buffers are filled within an unsafe section. This 892 + * prevents NBCON_PRIO_NORMAL and NBCON_PRIO_EMERGENCY from 893 + * clobbering each other. 894 + */ 895 + 896 + if (!nbcon_context_enter_unsafe(ctxt)) 897 + return false; 898 + 899 + ctxt->backlog = printk_get_next_message(&pmsg, ctxt->seq, is_extended, true); 900 + if (!ctxt->backlog) 901 + return nbcon_context_exit_unsafe(ctxt); 902 + 903 + /* 904 + * @con->dropped is not protected in case of an unsafe hostile 905 + * takeover. In that situation the update can be racy so 906 + * annotate it accordingly. 907 + */ 908 + con_dropped = data_race(READ_ONCE(con->dropped)); 909 + 910 + dropped = con_dropped + pmsg.dropped; 911 + if (dropped && !is_extended) 912 + console_prepend_dropped(&pmsg, dropped); 913 + 914 + if (!nbcon_context_exit_unsafe(ctxt)) 915 + return false; 916 + 917 + /* For skipped records just update seq/dropped in @con. */ 918 + if (pmsg.outbuf_len == 0) 919 + goto update_con; 920 + 921 + /* Initialize the write context for driver callbacks. */ 922 + wctxt->outbuf = &pmsg.pbufs->outbuf[0]; 923 + wctxt->len = pmsg.outbuf_len; 924 + nbcon_state_read(con, &cur); 925 + wctxt->unsafe_takeover = cur.unsafe_takeover; 926 + 927 + if (con->write_atomic) { 928 + done = con->write_atomic(con, wctxt); 929 + } else { 930 + nbcon_context_release(ctxt); 931 + WARN_ON_ONCE(1); 932 + done = false; 933 + } 934 + 935 + /* If not done, the emit was aborted. */ 936 + if (!done) 937 + return false; 938 + 939 + /* 940 + * Since any dropped message was successfully output, reset the 941 + * dropped count for the console. 942 + */ 943 + dropped = 0; 944 + update_con: 945 + /* 946 + * The dropped count and the sequence number are updated within an 947 + * unsafe section. This limits update races to the panic context and 948 + * allows the panic context to win. 949 + */ 950 + 951 + if (!nbcon_context_enter_unsafe(ctxt)) 952 + return false; 953 + 954 + if (dropped != con_dropped) { 955 + /* Counterpart to the READ_ONCE() above. */ 956 + WRITE_ONCE(con->dropped, dropped); 957 + } 958 + 959 + nbcon_seq_try_update(ctxt, pmsg.seq + 1); 960 + 961 + return nbcon_context_exit_unsafe(ctxt); 962 + } 963 + 964 + /** 965 + * nbcon_alloc - Allocate buffers needed by the nbcon console 966 + * @con: Console to allocate buffers for 967 + * 968 + * Return: True on success. False otherwise and the console cannot 969 + * be used. 970 + * 971 + * This is not part of nbcon_init() because buffer allocation must 972 + * be performed earlier in the console registration process. 973 + */ 974 + bool nbcon_alloc(struct console *con) 975 + { 976 + if (con->flags & CON_BOOT) { 977 + /* 978 + * Boot console printing is synchronized with legacy console 979 + * printing, so boot consoles can share the same global printk 980 + * buffers. 981 + */ 982 + con->pbufs = &printk_shared_pbufs; 983 + } else { 984 + con->pbufs = kmalloc(sizeof(*con->pbufs), GFP_KERNEL); 985 + if (!con->pbufs) { 986 + con_printk(KERN_ERR, con, "failed to allocate printing buffer\n"); 987 + return false; 988 + } 989 + } 990 + 991 + return true; 992 + } 993 + 994 + /** 995 + * nbcon_init - Initialize the nbcon console specific data 996 + * @con: Console to initialize 997 + * 998 + * nbcon_alloc() *must* be called and succeed before this function 999 + * is called. 1000 + * 1001 + * This function expects that the legacy @con->seq has been set. 1002 + */ 1003 + void nbcon_init(struct console *con) 1004 + { 1005 + struct nbcon_state state = { }; 1006 + 1007 + /* nbcon_alloc() must have been called and successful! */ 1008 + BUG_ON(!con->pbufs); 1009 + 1010 + nbcon_seq_force(con, con->seq); 1011 + nbcon_state_set(con, &state); 1012 + } 1013 + 1014 + /** 1015 + * nbcon_free - Free and cleanup the nbcon console specific data 1016 + * @con: Console to free/cleanup nbcon data 1017 + */ 1018 + void nbcon_free(struct console *con) 1019 + { 1020 + struct nbcon_state state = { }; 1021 + 1022 + nbcon_state_set(con, &state); 1023 + 1024 + /* Boot consoles share global printk buffers. */ 1025 + if (!(con->flags & CON_BOOT)) 1026 + kfree(con->pbufs); 1027 + 1028 + con->pbufs = NULL; 1029 + }

+86 -64

kernel/printk/printk.c

··· 102 102 */ 103 103 int __read_mostly suppress_printk; 104 104 105 - /* 106 - * During panic, heavy printk by other CPUs can delay the 107 - * panic and risk deadlock on console resources. 108 - */ 109 - static int __read_mostly suppress_panic_printk; 110 - 111 105 #ifdef CONFIG_LOCKDEP 112 106 static struct lockdep_map console_lock_dep_map = { 113 107 .name = "console_lock" ··· 439 445 static DEFINE_MUTEX(syslog_lock); 440 446 441 447 #ifdef CONFIG_PRINTK 448 + /* 449 + * During panic, heavy printk by other CPUs can delay the 450 + * panic and risk deadlock on console resources. 451 + */ 452 + static int __read_mostly suppress_panic_printk; 453 + 442 454 DECLARE_WAIT_QUEUE_HEAD(log_wait); 443 455 /* All 3 protected by @syslog_lock. */ 444 456 /* the next printk record to read by syslog(READ) or /proc/kmsg */ ··· 494 494 495 495 static struct printk_ringbuffer printk_rb_dynamic; 496 496 497 - static struct printk_ringbuffer *prb = &printk_rb_static; 497 + struct printk_ringbuffer *prb = &printk_rb_static; 498 498 499 499 /* 500 500 * We cannot access per-CPU data (e.g. per-CPU flush irq_work) before ··· 697 697 out: 698 698 return len; 699 699 } 700 - 701 - static bool printk_get_next_message(struct printk_message *pmsg, u64 seq, 702 - bool is_extended, bool may_supress); 703 700 704 701 /* /dev/kmsg - userspace message inject/listen interface */ 705 702 struct devkmsg_user { ··· 1666 1669 1667 1670 prb_rec_init_rd(&r, &info, text, PRINTK_MESSAGE_MAX); 1668 1671 1669 - len = 0; 1670 1672 prb_for_each_record(seq, prb, seq, &r) { 1671 1673 int textlen; 1672 1674 ··· 2345 2349 2346 2350 static u64 syslog_seq; 2347 2351 2348 - static size_t record_print_text(const struct printk_record *r, 2349 - bool syslog, bool time) 2350 - { 2351 - return 0; 2352 - } 2353 - static ssize_t info_print_ext_header(char *buf, size_t size, 2354 - struct printk_info *info) 2355 - { 2356 - return 0; 2357 - } 2358 - static ssize_t msg_print_ext_body(char *buf, size_t size, 2359 - char *text, size_t text_len, 2360 - struct dev_printk_info *dev_info) { return 0; } 2361 - static void console_lock_spinning_enable(void) { } 2362 - static int console_lock_spinning_disable_and_check(int cookie) { return 0; } 2363 - static bool suppress_message_printing(int level) { return false; } 2364 2352 static bool pr_flush(int timeout_ms, bool reset_on_progress) { return true; } 2365 2353 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; } 2366 2354 ··· 2698 2718 up_console_sem(); 2699 2719 } 2700 2720 2721 + #ifdef CONFIG_PRINTK 2722 + 2701 2723 /* 2702 2724 * Prepend the message in @pmsg->pbufs->outbuf with a "dropped message". This 2703 2725 * is achieved by shifting the existing message over and inserting the dropped ··· 2714 2732 * 2715 2733 * If @pmsg->pbufs->outbuf is modified, @pmsg->outbuf_len is updated. 2716 2734 */ 2717 - #ifdef CONFIG_PRINTK 2718 - static void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) 2735 + void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) 2719 2736 { 2720 2737 struct printk_buffers *pbufs = pmsg->pbufs; 2721 2738 const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf); ··· 2745 2764 memcpy(outbuf, scratchbuf, len); 2746 2765 pmsg->outbuf_len += len; 2747 2766 } 2748 - #else 2749 - #define console_prepend_dropped(pmsg, dropped) 2750 - #endif /* CONFIG_PRINTK */ 2751 2767 2752 2768 /* 2753 2769 * Read and format the specified record (or a later record if the specified ··· 2765 2787 * of @pmsg are valid. (See the documentation of struct printk_message 2766 2788 * for information about the @pmsg fields.) 2767 2789 */ 2768 - static bool printk_get_next_message(struct printk_message *pmsg, u64 seq, 2769 - bool is_extended, bool may_suppress) 2790 + bool printk_get_next_message(struct printk_message *pmsg, u64 seq, 2791 + bool is_extended, bool may_suppress) 2770 2792 { 2771 2793 static int panic_console_dropped; 2772 2794 ··· 2825 2847 } 2826 2848 2827 2849 /* 2850 + * Used as the printk buffers for non-panic, serialized console printing. 2851 + * This is for legacy (!CON_NBCON) as well as all boot (CON_BOOT) consoles. 2852 + * Its usage requires the console_lock held. 2853 + */ 2854 + struct printk_buffers printk_shared_pbufs; 2855 + 2856 + /* 2828 2857 * Print one record for the given console. The record printed is whatever 2829 2858 * record is the next available record for the given console. 2830 2859 * ··· 2848 2863 */ 2849 2864 static bool console_emit_next_record(struct console *con, bool *handover, int cookie) 2850 2865 { 2851 - static struct printk_buffers pbufs; 2852 - 2853 2866 bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED; 2854 - char *outbuf = &pbufs.outbuf[0]; 2867 + char *outbuf = &printk_shared_pbufs.outbuf[0]; 2855 2868 struct printk_message pmsg = { 2856 - .pbufs = &pbufs, 2869 + .pbufs = &printk_shared_pbufs, 2857 2870 }; 2858 2871 unsigned long flags; 2859 2872 ··· 2901 2918 skip: 2902 2919 return true; 2903 2920 } 2921 + 2922 + #else 2923 + 2924 + static bool console_emit_next_record(struct console *con, bool *handover, int cookie) 2925 + { 2926 + *handover = false; 2927 + return false; 2928 + } 2929 + 2930 + #endif /* CONFIG_PRINTK */ 2904 2931 2905 2932 /* 2906 2933 * Print out all remaining records to all consoles. ··· 3156 3163 3157 3164 if (mode == CONSOLE_REPLAY_ALL) { 3158 3165 struct console *c; 3166 + short flags; 3159 3167 int cookie; 3160 3168 u64 seq; 3161 3169 ··· 3164 3170 3165 3171 cookie = console_srcu_read_lock(); 3166 3172 for_each_console_srcu(c) { 3167 - /* 3168 - * This is an unsynchronized assignment, but the 3169 - * kernel is in "hope and pray" mode anyway. 3170 - */ 3171 - c->seq = seq; 3173 + flags = console_srcu_read_flags(c); 3174 + 3175 + if (flags & CON_NBCON) { 3176 + nbcon_seq_force(c, seq); 3177 + } else { 3178 + /* 3179 + * This is an unsynchronized assignment. On 3180 + * panic legacy consoles are only best effort. 3181 + */ 3182 + c->seq = seq; 3183 + } 3172 3184 } 3173 3185 console_srcu_read_unlock(cookie); 3174 3186 } ··· 3326 3326 newcon->flags |= CON_CONSDEV; 3327 3327 } 3328 3328 3329 - #define con_printk(lvl, con, fmt, ...) \ 3330 - printk(lvl pr_fmt("%sconsole [%s%d] " fmt), \ 3331 - (con->flags & CON_BOOT) ? "boot" : "", \ 3332 - con->name, con->index, ##__VA_ARGS__) 3333 - 3334 3329 static void console_init_seq(struct console *newcon, bool bootcon_registered) 3335 3330 { 3336 3331 struct console *con; ··· 3439 3444 goto unlock; 3440 3445 } 3441 3446 3447 + if (newcon->flags & CON_NBCON) { 3448 + /* 3449 + * Ensure the nbcon console buffers can be allocated 3450 + * before modifying any global data. 3451 + */ 3452 + if (!nbcon_alloc(newcon)) 3453 + goto unlock; 3454 + } 3455 + 3442 3456 /* 3443 3457 * See if we want to enable this console driver by default. 3444 3458 * ··· 3475 3471 err = try_enable_preferred_console(newcon, false); 3476 3472 3477 3473 /* printk() messages are not printed to the Braille console. */ 3478 - if (err || newcon->flags & CON_BRL) 3474 + if (err || newcon->flags & CON_BRL) { 3475 + if (newcon->flags & CON_NBCON) 3476 + nbcon_free(newcon); 3479 3477 goto unlock; 3478 + } 3480 3479 3481 3480 /* 3482 3481 * If we have a bootconsole, and are switching to a real console, ··· 3494 3487 3495 3488 newcon->dropped = 0; 3496 3489 console_init_seq(newcon, bootcon_registered); 3490 + 3491 + if (newcon->flags & CON_NBCON) 3492 + nbcon_init(newcon); 3497 3493 3498 3494 /* 3499 3495 * Put this console in the list - keep the ··· 3588 3578 * exit/cleanup routines can be performed safely. 3589 3579 */ 3590 3580 synchronize_srcu(&console_srcu); 3581 + 3582 + if (console->flags & CON_NBCON) 3583 + nbcon_free(console); 3591 3584 3592 3585 console_sysfs_notify(); 3593 3586 ··· 3741 3728 /* If @con is specified, only wait for that console. Otherwise wait for all. */ 3742 3729 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) 3743 3730 { 3744 - int remaining = timeout_ms; 3731 + unsigned long timeout_jiffies = msecs_to_jiffies(timeout_ms); 3732 + unsigned long remaining_jiffies = timeout_jiffies; 3745 3733 struct console *c; 3746 3734 u64 last_diff = 0; 3747 3735 u64 printk_seq; 3736 + short flags; 3748 3737 int cookie; 3749 3738 u64 diff; 3750 3739 u64 seq; ··· 3760 3745 console_unlock(); 3761 3746 3762 3747 for (;;) { 3748 + unsigned long begin_jiffies; 3749 + unsigned long slept_jiffies; 3750 + 3763 3751 diff = 0; 3764 3752 3765 3753 /* ··· 3777 3759 for_each_console_srcu(c) { 3778 3760 if (con && con != c) 3779 3761 continue; 3762 + 3763 + flags = console_srcu_read_flags(c); 3764 + 3780 3765 /* 3781 3766 * If consoles are not usable, it cannot be expected 3782 3767 * that they make forward progress, so only increment ··· 3787 3766 */ 3788 3767 if (!console_is_usable(c)) 3789 3768 continue; 3790 - printk_seq = c->seq; 3769 + 3770 + if (flags & CON_NBCON) { 3771 + printk_seq = nbcon_seq_read(c); 3772 + } else { 3773 + printk_seq = c->seq; 3774 + } 3775 + 3791 3776 if (printk_seq < seq) 3792 3777 diff += seq - printk_seq; 3793 3778 } 3794 3779 console_srcu_read_unlock(cookie); 3795 3780 3796 3781 if (diff != last_diff && reset_on_progress) 3797 - remaining = timeout_ms; 3782 + remaining_jiffies = timeout_jiffies; 3798 3783 3799 3784 console_unlock(); 3800 3785 3801 3786 /* Note: @diff is 0 if there are no usable consoles. */ 3802 - if (diff == 0 || remaining == 0) 3787 + if (diff == 0 || remaining_jiffies == 0) 3803 3788 break; 3804 3789 3805 - if (remaining < 0) { 3806 - /* no timeout limit */ 3807 - msleep(100); 3808 - } else if (remaining < 100) { 3809 - msleep(remaining); 3810 - remaining = 0; 3811 - } else { 3812 - msleep(100); 3813 - remaining -= 100; 3814 - } 3790 + /* msleep(1) might sleep much longer. Check time by jiffies. */ 3791 + begin_jiffies = jiffies; 3792 + msleep(1); 3793 + slept_jiffies = jiffies - begin_jiffies; 3794 + 3795 + remaining_jiffies -= min(slept_jiffies, remaining_jiffies); 3815 3796 3816 3797 last_diff = diff; 3817 3798 } ··· 4217 4194 4218 4195 prb_rec_init_rd(&r, &info, buf, size); 4219 4196 4220 - len = 0; 4221 4197 prb_for_each_record(seq, prb, seq, &r) { 4222 4198 if (r.info->seq >= iter->next_seq) 4223 4199 break;

+12 -13

lib/vsprintf.c

··· 60 60 bool no_hash_pointers __ro_after_init; 61 61 EXPORT_SYMBOL_GPL(no_hash_pointers); 62 62 63 - static noinline unsigned long long simple_strntoull(const char *startp, size_t max_chars, char **endp, unsigned int base) 63 + noinline 64 + static unsigned long long simple_strntoull(const char *startp, char **endp, unsigned int base, size_t max_chars) 64 65 { 65 66 const char *cp; 66 67 unsigned long long result = 0ULL; ··· 96 95 noinline 97 96 unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base) 98 97 { 99 - return simple_strntoull(cp, INT_MAX, endp, base); 98 + return simple_strntoull(cp, endp, base, INT_MAX); 100 99 } 101 100 EXPORT_SYMBOL(simple_strtoull); 102 101 ··· 131 130 } 132 131 EXPORT_SYMBOL(simple_strtol); 133 132 134 - static long long simple_strntoll(const char *cp, size_t max_chars, char **endp, 135 - unsigned int base) 133 + noinline 134 + static long long simple_strntoll(const char *cp, char **endp, unsigned int base, size_t max_chars) 136 135 { 137 136 /* 138 137 * simple_strntoull() safely handles receiving max_chars==0 in the ··· 141 140 * and the content of *cp is irrelevant. 142 141 */ 143 142 if (*cp == '-' && max_chars > 0) 144 - return -simple_strntoull(cp + 1, max_chars - 1, endp, base); 143 + return -simple_strntoull(cp + 1, endp, base, max_chars - 1); 145 144 146 - return simple_strntoull(cp, max_chars, endp, base); 145 + return simple_strntoull(cp, endp, base, max_chars); 147 146 } 148 147 149 148 /** ··· 156 155 */ 157 156 long long simple_strtoll(const char *cp, char **endp, unsigned int base) 158 157 { 159 - return simple_strntoll(cp, INT_MAX, endp, base); 158 + return simple_strntoll(cp, endp, base, INT_MAX); 160 159 } 161 160 EXPORT_SYMBOL(simple_strtoll); 162 161 ··· 3649 3648 break; 3650 3649 3651 3650 if (is_sign) 3652 - val.s = simple_strntoll(str, 3653 - field_width >= 0 ? field_width : INT_MAX, 3654 - &next, base); 3651 + val.s = simple_strntoll(str, &next, base, 3652 + field_width >= 0 ? field_width : INT_MAX); 3655 3653 else 3656 - val.u = simple_strntoull(str, 3657 - field_width >= 0 ? field_width : INT_MAX, 3658 - &next, base); 3654 + val.u = simple_strntoull(str, &next, base, 3655 + field_width >= 0 ? field_width : INT_MAX); 3659 3656 3660 3657 switch (qualifier) { 3661 3658 case 'H': /* that's 'hh' in format */

Configure Feed

Configure Feed