tracing: Support to dump instance traces by ftrace_dump_on_oops

+21 -5

Documentation/admin-guide/kernel-parameters.txt

··· 1572 1572 The above will cause the "foo" tracing instance to trigger 1573 1573 a snapshot at the end of boot up. 1574 1574 1575 - ftrace_dump_on_oops[=orig_cpu] 1575 + ftrace_dump_on_oops[=2(orig_cpu) | =<instance>][,<instance> | 1576 + ,<instance>=2(orig_cpu)] 1576 1577 [FTRACE] will dump the trace buffers on oops. 1577 - If no parameter is passed, ftrace will dump 1578 - buffers of all CPUs, but if you pass orig_cpu, it will 1579 - dump only the buffer of the CPU that triggered the 1580 - oops. 1578 + If no parameter is passed, ftrace will dump global 1579 + buffers of all CPUs, if you pass 2 or orig_cpu, it 1580 + will dump only the buffer of the CPU that triggered 1581 + the oops, or the specific instance will be dumped if 1582 + its name is passed. Multiple instance dump is also 1583 + supported, and instances are separated by commas. Each 1584 + instance supports only dump on CPU that triggered the 1585 + oops by passing 2 or orig_cpu to it. 1586 + 1587 + ftrace_dump_on_oops=foo=orig_cpu 1588 + 1589 + The above will dump only the buffer of "foo" instance 1590 + on CPU that triggered the oops. 1591 + 1592 + ftrace_dump_on_oops,foo,bar=orig_cpu 1593 + 1594 + The above will dump global buffer on all CPUs, the 1595 + buffer of "foo" instance on all CPUs and the buffer 1596 + of "bar" instance on CPU that triggered the oops. 1581 1597 1582 1598 ftrace_filter=[function-list] 1583 1599 [FTRACE] Limit the functions traced by the function

+23 -5

Documentation/admin-guide/sysctl/kernel.rst

··· 296 296 the console. This is very useful for capturing traces that lead to 297 297 crashes and outputting them to a serial console. 298 298 299 - = =================================================== 300 - 0 Disabled (default). 301 - 1 Dump buffers of all CPUs. 302 - 2 Dump the buffer of the CPU that triggered the oops. 303 - = =================================================== 299 + ======================= =========================================== 300 + 0 Disabled (default). 301 + 1 Dump buffers of all CPUs. 302 + 2(orig_cpu) Dump the buffer of the CPU that triggered the 303 + oops. 304 + <instance> Dump the specific instance buffer on all CPUs. 305 + <instance>=2(orig_cpu) Dump the specific instance buffer on the CPU 306 + that triggered the oops. 307 + ======================= =========================================== 304 308 309 + Multiple instance dump is also supported, and instances are separated 310 + by commas. If global buffer also needs to be dumped, please specify 311 + the dump mode (1/2/orig_cpu) first for global buffer. 312 + 313 + So for example to dump "foo" and "bar" instance buffer on all CPUs, 314 + user can:: 315 + 316 + echo "foo,bar" > /proc/sys/kernel/ftrace_dump_on_oops 317 + 318 + To dump global buffer and "foo" instance buffer on all 319 + CPUs along with the "bar" instance buffer on CPU that triggered the 320 + oops, user can:: 321 + 322 + echo "1,foo,bar=2" > /proc/sys/kernel/ftrace_dump_on_oops 305 323 306 324 ftrace_enabled, stack_tracer_enabled 307 325 ====================================

+3 -1

include/linux/ftrace.h

··· 1151 1151 #ifdef CONFIG_TRACING 1152 1152 enum ftrace_dump_mode; 1153 1153 1154 - extern enum ftrace_dump_mode ftrace_dump_on_oops; 1154 + #define MAX_TRACER_SIZE 100 1155 + extern char ftrace_dump_on_oops[]; 1156 + extern int ftrace_dump_on_oops_enabled(void); 1155 1157 extern int tracepoint_printk; 1156 1158 1157 1159 extern void disable_trace_on_warning(void);

+1

include/linux/kernel.h

··· 215 215 DUMP_NONE, 216 216 DUMP_ALL, 217 217 DUMP_ORIG, 218 + DUMP_PARAM, 218 219 }; 219 220 220 221 #ifdef CONFIG_TRACING

+2 -2

kernel/sysctl.c

··· 1710 1710 { 1711 1711 .procname = "ftrace_dump_on_oops", 1712 1712 .data = &ftrace_dump_on_oops, 1713 - .maxlen = sizeof(int), 1713 + .maxlen = MAX_TRACER_SIZE, 1714 1714 .mode = 0644, 1715 - .proc_handler = proc_dointvec, 1715 + .proc_handler = proc_dostring, 1716 1716 }, 1717 1717 { 1718 1718 .procname = "traceoff_on_warning",

+116 -40

kernel/trace/trace.c

··· 130 130 * /proc/sys/kernel/ftrace_dump_on_oops 131 131 * Set 1 if you want to dump buffers of all CPUs 132 132 * Set 2 if you want to dump the buffer of the CPU that triggered oops 133 + * Set instance name if you want to dump the specific trace instance 134 + * Multiple instance dump is also supported, and instances are seperated 135 + * by commas. 133 136 */ 134 - 135 - enum ftrace_dump_mode ftrace_dump_on_oops; 137 + /* Set to string format zero to disable by default */ 138 + char ftrace_dump_on_oops[MAX_TRACER_SIZE] = "0"; 136 139 137 140 /* When set, tracing will stop when a WARN*() is hit */ 138 141 int __disable_trace_on_warning; ··· 181 178 struct trace_buffer *buffer, 182 179 unsigned int trace_ctx); 183 180 184 - #define MAX_TRACER_SIZE 100 185 181 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; 186 182 static char *default_bootup_tracer; 187 183 ··· 203 201 } 204 202 __setup("ftrace=", set_cmdline_ftrace); 205 203 204 + int ftrace_dump_on_oops_enabled(void) 205 + { 206 + if (!strcmp("0", ftrace_dump_on_oops)) 207 + return 0; 208 + else 209 + return 1; 210 + } 211 + 206 212 static int __init set_ftrace_dump_on_oops(char *str) 207 213 { 208 - if (*str++ != '=' || !*str || !strcmp("1", str)) { 209 - ftrace_dump_on_oops = DUMP_ALL; 214 + if (!*str) { 215 + strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE); 210 216 return 1; 211 217 } 212 218 213 - if (!strcmp("orig_cpu", str) || !strcmp("2", str)) { 214 - ftrace_dump_on_oops = DUMP_ORIG; 215 - return 1; 216 - } 219 + if (*str == ',') { 220 + strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE); 221 + strscpy(ftrace_dump_on_oops + 1, str, MAX_TRACER_SIZE - 1); 222 + return 1; 223 + } 217 224 218 - return 0; 225 + if (*str++ == '=') { 226 + strscpy(ftrace_dump_on_oops, str, MAX_TRACER_SIZE); 227 + return 1; 228 + } 229 + 230 + return 0; 219 231 } 220 232 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops); 221 233 ··· 9848 9832 static int trace_die_panic_handler(struct notifier_block *self, 9849 9833 unsigned long ev, void *unused) 9850 9834 { 9851 - if (!ftrace_dump_on_oops) 9835 + if (!ftrace_dump_on_oops_enabled()) 9852 9836 return NOTIFY_DONE; 9853 9837 9854 9838 /* The die notifier requires DIE_OOPS to trigger */ 9855 9839 if (self == &trace_die_notifier && ev != DIE_OOPS) 9856 9840 return NOTIFY_DONE; 9857 9841 9858 - ftrace_dump(ftrace_dump_on_oops); 9842 + ftrace_dump(DUMP_PARAM); 9859 9843 9860 9844 return NOTIFY_DONE; 9861 9845 } ··· 9896 9880 trace_seq_init(s); 9897 9881 } 9898 9882 9899 - void trace_init_global_iter(struct trace_iterator *iter) 9883 + static void trace_init_iter(struct trace_iterator *iter, struct trace_array *tr) 9900 9884 { 9901 - iter->tr = &global_trace; 9885 + iter->tr = tr; 9902 9886 iter->trace = iter->tr->current_trace; 9903 9887 iter->cpu_file = RING_BUFFER_ALL_CPUS; 9904 - iter->array_buffer = &global_trace.array_buffer; 9888 + iter->array_buffer = &tr->array_buffer; 9905 9889 9906 9890 if (iter->trace && iter->trace->open) 9907 9891 iter->trace->open(iter); ··· 9921 9905 iter->fmt_size = STATIC_FMT_BUF_SIZE; 9922 9906 } 9923 9907 9924 - void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) 9908 + void trace_init_global_iter(struct trace_iterator *iter) 9909 + { 9910 + trace_init_iter(iter, &global_trace); 9911 + } 9912 + 9913 + static void ftrace_dump_one(struct trace_array *tr, enum ftrace_dump_mode dump_mode) 9925 9914 { 9926 9915 /* use static because iter can be a bit big for the stack */ 9927 9916 static struct trace_iterator iter; 9928 - static atomic_t dump_running; 9929 - struct trace_array *tr = &global_trace; 9930 9917 unsigned int old_userobj; 9931 9918 unsigned long flags; 9932 9919 int cnt = 0, cpu; 9933 - 9934 - /* Only allow one dump user at a time. */ 9935 - if (atomic_inc_return(&dump_running) != 1) { 9936 - atomic_dec(&dump_running); 9937 - return; 9938 - } 9939 9920 9940 9921 /* 9941 9922 * Always turn off tracing when we dump. ··· 9942 9929 * If the user does a sysrq-z, then they can re-enable 9943 9930 * tracing with echo 1 > tracing_on. 9944 9931 */ 9945 - tracing_off(); 9932 + tracer_tracing_off(tr); 9946 9933 9947 9934 local_irq_save(flags); 9948 9935 9949 9936 /* Simulate the iterator */ 9950 - trace_init_global_iter(&iter); 9937 + trace_init_iter(&iter, tr); 9951 9938 9952 9939 for_each_tracing_cpu(cpu) { 9953 9940 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); ··· 9958 9945 /* don't look at user memory in panic mode */ 9959 9946 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ; 9960 9947 9961 - switch (oops_dump_mode) { 9962 - case DUMP_ALL: 9963 - iter.cpu_file = RING_BUFFER_ALL_CPUS; 9964 - break; 9965 - case DUMP_ORIG: 9948 + if (dump_mode == DUMP_ORIG) 9966 9949 iter.cpu_file = raw_smp_processor_id(); 9967 - break; 9968 - case DUMP_NONE: 9969 - goto out_enable; 9970 - default: 9971 - printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); 9950 + else 9972 9951 iter.cpu_file = RING_BUFFER_ALL_CPUS; 9973 - } 9974 9952 9975 - printk(KERN_TRACE "Dumping ftrace buffer:\n"); 9953 + if (tr == &global_trace) 9954 + printk(KERN_TRACE "Dumping ftrace buffer:\n"); 9955 + else 9956 + printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name); 9976 9957 9977 9958 /* Did function tracer already get disabled? */ 9978 9959 if (ftrace_is_dead()) { ··· 10008 10001 else 10009 10002 printk(KERN_TRACE "---------------------------------\n"); 10010 10003 10011 - out_enable: 10012 10004 tr->trace_flags |= old_userobj; 10013 10005 10014 10006 for_each_tracing_cpu(cpu) { 10015 10007 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); 10016 10008 } 10017 - atomic_dec(&dump_running); 10018 10009 local_irq_restore(flags); 10010 + } 10011 + 10012 + static void ftrace_dump_by_param(void) 10013 + { 10014 + bool first_param = true; 10015 + char dump_param[MAX_TRACER_SIZE]; 10016 + char *buf, *token, *inst_name; 10017 + struct trace_array *tr; 10018 + 10019 + strscpy(dump_param, ftrace_dump_on_oops, MAX_TRACER_SIZE); 10020 + buf = dump_param; 10021 + 10022 + while ((token = strsep(&buf, ",")) != NULL) { 10023 + if (first_param) { 10024 + first_param = false; 10025 + if (!strcmp("0", token)) 10026 + continue; 10027 + else if (!strcmp("1", token)) { 10028 + ftrace_dump_one(&global_trace, DUMP_ALL); 10029 + continue; 10030 + } 10031 + else if (!strcmp("2", token) || 10032 + !strcmp("orig_cpu", token)) { 10033 + ftrace_dump_one(&global_trace, DUMP_ORIG); 10034 + continue; 10035 + } 10036 + } 10037 + 10038 + inst_name = strsep(&token, "="); 10039 + tr = trace_array_find(inst_name); 10040 + if (!tr) { 10041 + printk(KERN_TRACE "Instance %s not found\n", inst_name); 10042 + continue; 10043 + } 10044 + 10045 + if (token && (!strcmp("2", token) || 10046 + !strcmp("orig_cpu", token))) 10047 + ftrace_dump_one(tr, DUMP_ORIG); 10048 + else 10049 + ftrace_dump_one(tr, DUMP_ALL); 10050 + } 10051 + } 10052 + 10053 + void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) 10054 + { 10055 + static atomic_t dump_running; 10056 + 10057 + /* Only allow one dump user at a time. */ 10058 + if (atomic_inc_return(&dump_running) != 1) { 10059 + atomic_dec(&dump_running); 10060 + return; 10061 + } 10062 + 10063 + switch (oops_dump_mode) { 10064 + case DUMP_ALL: 10065 + ftrace_dump_one(&global_trace, DUMP_ALL); 10066 + break; 10067 + case DUMP_ORIG: 10068 + ftrace_dump_one(&global_trace, DUMP_ORIG); 10069 + break; 10070 + case DUMP_PARAM: 10071 + ftrace_dump_by_param(); 10072 + break; 10073 + case DUMP_NONE: 10074 + break; 10075 + default: 10076 + printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); 10077 + ftrace_dump_one(&global_trace, DUMP_ALL); 10078 + } 10079 + 10080 + atomic_dec(&dump_running); 10019 10081 } 10020 10082 EXPORT_SYMBOL_GPL(ftrace_dump); 10021 10083

+1 -1

kernel/trace/trace_selftest.c

··· 768 768 if (unlikely(++graph_hang_thresh > GRAPH_MAX_FUNC_TEST)) { 769 769 ftrace_graph_stop(); 770 770 printk(KERN_WARNING "BUG: Function graph tracer hang!\n"); 771 - if (ftrace_dump_on_oops) { 771 + if (ftrace_dump_on_oops_enabled()) { 772 772 ftrace_dump(DUMP_ALL); 773 773 /* ftrace_dump() disables tracing */ 774 774 tracing_on();

Configure Feed

Configure Feed